diff --git a/.gitignore b/.gitignore index be2ad9afb..004f7d759 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,7 @@ scripts/training/train-model.perl dist bin previous.sh +contrib/other-builds/*.xcodeproj/project.xcworkspace/ +contrib/other-builds/*.xcodeproj/xcuserdata/ +*/*.xcodeproj/project.xcworkspace +*/*.xcodeproj/xcuserdata diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..0470c63fd --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "regression-testing/tests"] + path = regression-testing/tests + url = ../moses-regression-tests.git diff --git a/Jamroot b/Jamroot index 98b2e2442..927ccc577 100644 --- a/Jamroot +++ b/Jamroot @@ -15,13 +15,15 @@ #Note that, like language models, this is the --prefix where the library was #installed, not some executable within the library. # +#Compact phrase table and compact lexical reordering table +#--with-cmph=/path/to/cmph +# #Thread-caching malloc (optional): #--with-tcmalloc # #REGRESSION TESTING #--with-regtest=/path/to/moses-reg-test-data # -# #INSTALLATION #--prefix=/path/to/prefix sets the install prefix [default is source root]. #--bindir=/path/to/prefix/bin sets the bin directory [PREFIX/bin] @@ -29,6 +31,7 @@ #--includedir=/path/to/prefix/include installs headers. # Does not install if missing. No argument defaults to PREFIX/include . #--install-scripts=/path/to/scripts copies scripts into a directory. +# Does not install if missing. No argument defaults to PREFIX/scripts . #--git appends the git revision to the prefix directory. # # @@ -41,7 +44,9 @@ # variant=release|debug|profile builds optimized (default), for debug, or for # profiling # -# link=static|shared controls linking (default static) +# link=static|shared controls preferred linking (default static) +# --static forces static linking (the default will fall +# back to shared) # # debug-symbols=on|off include (default) or exclude debugging # information also known as -g @@ -50,6 +55,9 @@ # --enable-boost-pool uses Boost pools for the memory SCFG table # # --enable-mpi switch on mpi +# --without-libsegfault does not link with libSegFault +# +# --max-kenlm-order maximum ngram order that kenlm can process (default 6) # #CONTROLLING THE BUILD #-a to build from scratch @@ -84,6 +92,10 @@ if [ option.get "enable-mpi" : : "yes" ] { requirements += [ option.get "notrace" : TRACE_ENABLE=1 ] ; requirements += [ option.get "enable-boost-pool" : : USE_BOOST_POOL ] ; +if [ option.get "with-cmph" ] { + requirements += HAVE_CMPH ; +} + project : default-build multi on @@ -99,12 +111,13 @@ project : requirements $(requirements) ; -build-projects util lm mert moses-cmd/src moses-chart-cmd/src mira scripts regression-testing ; +#Add directories here if you want their incidental targets too (i.e. tests). +build-projects util lm mert moses-cmd/src moses-chart-cmd/src mira scripts regression-testing ; -alias programs : lm//query lm//build_binary moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs mira//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score biconcor ; +alias programs : lm//query lm//build_binary lm//kenlm_max_order moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs mira//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score biconcor ; install-bin-libs programs ; -install-headers headers-base : [ glob-tree *.h *.hh : jam-files dist bin lib include kenlm moses ] : . ; +install-headers headers-base : [ path.glob-tree biconcor contrib lm mert misc moses-chart-cmd moses-cmd OnDiskPt phrase-extract symal util : *.hh *.h ] : . ; install-headers headers-moses : moses/src//headers-to-install : moses/src ; alias install : prefix-bin prefix-lib headers-base headers-moses ; diff --git a/OnDiskPt/TargetPhrase.cpp b/OnDiskPt/TargetPhrase.cpp index 4e6739fda..e7123af3b 100644 --- a/OnDiskPt/TargetPhrase.cpp +++ b/OnDiskPt/TargetPhrase.cpp @@ -240,9 +240,7 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vectorAddWord(*mosesWord); - delete mosesWord; + GetWord(pos).ConvertToMoses(outputFactors, vocab, ret->AddWord()); } // scores @@ -261,16 +259,12 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vectorSetAlignmentInfo(alignmentInfo, indicator); - Moses::Word *lhs = GetWord(GetSize() - 1).ConvertToMoses(Moses::Output, outputFactors, vocab); - ret->SetTargetLHS(*lhs); - delete lhs; + GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, ret->MutableTargetLHS()); // set source phrase Moses::Phrase mosesSP(Moses::Input); for (size_t pos = 0; pos < sp->GetSize(); ++pos) { - Moses::Word *mosesWord = sp->GetWord(pos).ConvertToMoses(Moses::Input, inputFactors, vocab); - mosesSP.AddWord(*mosesWord); - delete mosesWord; + sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord()); } ret->SetSourcePhrase(mosesSP); diff --git a/OnDiskPt/Word.cpp b/OnDiskPt/Word.cpp index 87d45818f..52e49d8d9 100644 --- a/OnDiskPt/Word.cpp +++ b/OnDiskPt/Word.cpp @@ -23,6 +23,9 @@ #include "../moses/src/Word.h" #include "Word.h" +#include "util/tokenize_piece.hh" +#include "util/exception.hh" + using namespace std; namespace OnDiskPt @@ -94,23 +97,21 @@ size_t Word::ReadFromFile(std::fstream &file) return memUsed; } -Moses::Word *Word::ConvertToMoses(Moses::FactorDirection direction - , const std::vector &outputFactorsVec - , const Vocab &vocab) const -{ +void Word::ConvertToMoses( + const std::vector &outputFactorsVec, + const Vocab &vocab, + Moses::Word &overwrite) const { Moses::FactorCollection &factorColl = Moses::FactorCollection::Instance(); - Moses::Word *ret = new Moses::Word(m_isNonTerminal); + overwrite = Moses::Word(m_isNonTerminal); - const string &str = vocab.GetString(m_vocabId); - vector toks = Moses::Tokenize(str, "|"); - for (size_t ind = 0; ind < toks.size(); ++ind) { - Moses::FactorType factorType = outputFactorsVec[ind]; - const Moses::Factor *factor = factorColl.AddFactor(direction, factorType, toks[ind]); - ret->SetFactor(factorType, factor); + // TODO: this conversion should have been done at load time. + util::TokenIter tok(vocab.GetString(m_vocabId), '|'); + + for (std::vector::const_iterator t = outputFactorsVec.begin(); t != outputFactorsVec.end(); ++t, ++tok) { + UTIL_THROW_IF(!tok, util::Exception, "Too few factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size()); + overwrite.SetFactor(*t, factorColl.AddFactor(*tok)); } - - return ret; - + UTIL_THROW_IF(tok, util::Exception, "Too many factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size()); } int Word::Compare(const Word &compare) const diff --git a/OnDiskPt/Word.h b/OnDiskPt/Word.h index 98ed109ab..8c65cf7e5 100644 --- a/OnDiskPt/Word.h +++ b/OnDiskPt/Word.h @@ -71,9 +71,10 @@ public: m_vocabId = vocabId; } - Moses::Word *ConvertToMoses(Moses::FactorDirection direction - , const std::vector &outputFactorsVec - , const Vocab &vocab) const; + void ConvertToMoses( + const std::vector &outputFactorsVec, + const Vocab &vocab, + Moses::Word &overwrite) const; virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const; diff --git a/bjam b/bjam index d1ac8a555..2b0232c8a 100755 --- a/bjam +++ b/bjam @@ -4,8 +4,8 @@ if bjam="$(which bjam 2>/dev/null)" && #exists [ ${#bjam} != 0 ] && #paranoia about which printing nothing then returning true ! grep UFIHGUFIHBDJKNCFZXAEVA "${bjam}" /dev/null && #bjam in path isn't this script - "${bjam}" --help >/dev/null 2>/dev/null && #bjam in path isn't broken (i.e. has boost-build) - "${bjam}" --version |grep "Boost.Build 201" >/dev/null 2>/dev/null #It's recent enough. + "${bjam}" --sanity-test 2>/dev/null |grep Sane >/dev/null && #The test in jam-files/sanity.jam passes + (cd jam-files/fail && ! "${bjam}") >/dev/null #Returns non-zero on failure then #Delegate to system bjam exec "${bjam}" "$@" diff --git a/contrib/combine-ptables/README.md b/contrib/combine-ptables/README.md new file mode 100644 index 000000000..b180f9202 --- /dev/null +++ b/contrib/combine-ptables/README.md @@ -0,0 +1,139 @@ +`combine-ptables.pl`: fill-up and other techniques of translation models combination. + +Author: +Arianna Bisazza bisazza[AT]fbk.eu + +ABOUT +----- +This tool implements "fill-up" and other operations that are useful to combine translation and reordering tables. +In the "fill-up" approach, the weights of out-domain data sources are estimated directly by MERT along with the +other model weights. + +This tool also supports linear interpolation, but weights must be provided by the user. +If you want to automatically estimate linear interpolation weights, use `contrib/tmcombine` instead. + + +REFERENCE +--------- +When using this script, please cite: +Arianna Bisazza, Nick Ruiz, and Marcello Federico. 2011. +"Fill-up versus Interpolation Methods for Phrase-based SMT Adaptation." +In International Workshop on Spoken Language Translation (IWSLT), San Francisco, CA. + + +FILL-UP +------- + +This combination technique is useful when the relevance of the models is known a priori, +e.g. when one is trained on in-domain data and the others on out-of-domain data. + +This mode preserves all the entries and scores coming from the first model, and adds +entries from the other models only if new. +If more than two tables are provided, each entry is taken only from the first table +that contains it. + +Moreover, a binary feature is added for each additional table to denote the provenance +of an entry. For in-domain entries, the binary features are all set to 1 (=exp(0)). +Entries coming from the 2nd table will have the 1st binary feature set to 2.718 (=exp(1)). + +This technique was proposed in the following works: + +Preslav Nakov. 2008. +"Improving English-Spanish Statistical Machine Translation: Experiments in Domain +Adaptation, Sentence Paraphrasing, Tokenization, and Recasing." +In Workshop on Statistical Machine Translation. + +Arianna Bisazza, Nick Ruiz, and Marcello Federico. 2011. +"Fill-up versus Interpolation Methods for Phrase-based SMT Adaptation." +In International Workshop on Spoken Language Translation (IWSLT), San Francisco, CA. + +The latter paper contains details about the present implementation as well as an empirical +evaluation of fill-up against other combination techniques. +Reordering model fill-up, cascaded fill-up and pruning criteria are also discussed in the +same paper. + +Among the findings of this paper, pruning new (out-of-domain) phrases with more than 4 +source words appeared to be beneficial on the Arabic-English TED task when combining the +in-domain models with MultiUn models. +This corresponds to the option: + `--newSourceMaxLength=4` + + +LINEAR INTERPOLATION +-------------------- + +This combination technique consists in linearly combining the feature values coming +from all tables. The combination weights should be provided by the user, otherwise +uniform weights are assumed. +When a phrase pair is absent from a table, a constant value (epsilon) is assumed for +the corresponding feature values. You may want to set your own epsilon. + +See [Bisazza et al. 2011] for an empirical comparison of uniformly weighted linear +interpolation against fill-up and decoding-time log-linear interpolation. In that paper, +epsilon was always set to 1e-06. + + +UNION +----- + +This combination technique creates the union of all phrase pairs and assigns to each +of them the concatenation of all tables scores. + + +INTERSECTION +------------ + +This combination technique creates the intersection of all phrase pairs: each phrase +pair that occurs in all phrase tables is output along with the feature vector taken +from the *first* table. +The intersection can be used to prune the reordering table in order to match the +entries of a corresponding pruned phrase table. + + +USAGE +----- + +Get statistics about overlap of entries: + `combine-ptables.pl --mode=stats ptable1 ptable2 ... ptableN > ptables-overlap-stats` + +Interpolate phrase tables... +- with uniform weights: + `combine-ptables.pl --mode=interp --phpenalty-at=4 ptable1 ptable2 ptable3 > interp-ptable.X` + +- with custom weights: + `combine-ptables.pl --mode=interp --phpenalty-at=4 --weights=0.8,0.1,0.1 ptable1 ptable2 ptable3 > interp-ptable.Y` + +- with custom epsilon: + `combine-ptables.pl --mode=interp --phpenalty-at=4 --epsilon=1e-05 ptable1 ptable2 ptable3 > interp-ptable.Z` + + +Fillup phrase tables... +- unpruned: + `combine-ptables.pl --mode=fillup ptable1 ptable2 ... ptableN > fillup-ptable` + +- pruned (new phrases only with max. 4 source words): + `combine-ptables.pl --mode=fillup --newSourceMaxLength=4 ptable1 ptable2 ... ptableN > fillup-ptable` + + +Given a pruned phrase table, prune the corresponding reordering table: + `combine-ptables.pl --mode=intersect1 reotable1-unpruned ptable1-pruned > reotable1-pruned` + + +NOTES +----- + +The script works only with textual (non-binarized) phrase or reordering tables +that were *previously sorted* with `LC_ALL=C sort` + +The resulting combined tables are also textual and need to binarized normally. + +The script combine-ptables.pl can be used on lexicalized reordering tables as well. + +Input tables can be gzipped. + +When integrating filled up models into a Moses system, remember to: + - specify the correct number of features (typically 6) under [ttable-file] in the configuration file `moses.ini` + - add a weight under [weight-t] in `moses.ini` + - if you binarize the models, provide the correct number of features to the command: + `$moses/bin/processPhraseTable -ttable 0 0 - -nscores $nbFeatures` + diff --git a/contrib/combine-ptables/combine-ptables.pl b/contrib/combine-ptables/combine-ptables.pl new file mode 100755 index 000000000..de9df7ec2 --- /dev/null +++ b/contrib/combine-ptables/combine-ptables.pl @@ -0,0 +1,425 @@ +#! /usr/bin/perl + +#****************************************************************************** +# Arianna Bisazza @ FBK-irst. March 2012 +#****************************************************************************** +# combine-ptables.pl : Combine Moses-style phrase tables, using different approaches + + +use strict; +use open ':utf8'; +binmode STDIN, ':utf8'; +binmode STDOUT, ':utf8'; + +use Getopt::Long "GetOptions"; + +sub main { +my $usage = " +USAGE +----- +combine-ptables.pl --mode=(interp|union|fillup|intersect1|stats) ptable1 ptable2 ... ptableN > combined-ptable +combine-ptables.pl --mode=intersect1 reotable-unpruned ptable-pruned > reotable-pruned +----- +# +# This scripts reads two or more *sorted* phrase tables and combines them in different modes. +# +# (Note: if present, word alignments are ignored). +# +# ---------------- +# OPTIONS +# ---------------- +# +# Required: +# --mode fillup: Each entry is taken only from the first table that contains it. +# A binary feature is added from each table except the first. +# interp: Linear interpolation. +# union: Union of entries, feature vectors are concatenated. +# intersect1: Intersection of entries, feature vectors taken from the first table. +# stats: Only compute some statistics about tables overlap. No table is produced. +# +# NOTE: if present, additional fields such as word alignment, phrase counts etc. are always +# taken from the first table. +# +# Generic options: +# --phpenalty=FLOAT Constant value for phrase penalty. Default is exp(1)=2.718 +# --phpenalty-at=N The (N+1)th score of each table is considered as phrase penalty with a constant value. +# In 'interp' mode, the corresponding feature is not interpolated but simply set to the constant. +# In 'union' mode, the ph.penalty (constant) is output only once, after all the other scores. +# By default, no score is considered as phrase penalty. +# +# +# Options for 'fillup': +# --newSourceMaxLength=INT Don't include \"new\" source phrases if longer than INT words. +# +# Options for 'interp': +# --weights=W1,W2,...WN Weights for interpolation. By default, uniform weights are applied. +# --epsilon=X Score to assume when a phrase pair is not contained in a table (in 'interp' and 'union' modes). +# Default epsilon is 1e-06. +# +# Options for 'union': +# +# +"; + +my $combination_mode = ''; +my $debug = ''; +my $weights_str = ''; +my $epsilon = 0.000001; +my $phPenalty = 2.718; # exp(1) +my $phPenalty_idx = -1; +my $delim= " ||| "; +my $delim_RE= ' \|\|\| '; +my $exp_one = 2.718; +my $exp_zero = 1; +my $newSourceMaxLength = -1; +my $help = ''; + +GetOptions ('debug' => \$debug, + 'mode=s' => \$combination_mode, + 'weights=s' => \$weights_str, + 'epsilon=f' => \$epsilon, + 'phpenalty=f' => \$phPenalty, + 'phpenalty-at=i' => \$phPenalty_idx, + 'newSourceMaxLength=i' => \$newSourceMaxLength, + 'help' => \$help); + +if($help) { die "$usage\n\n"; } + +if($combination_mode!~/(interp|union|fillup|intersect1|stats)/) {die "$usage\nUnknown combination mode!\n"}; + +if(@ARGV < 2) {die "$usage\n\n Please provide at least 2 tables to combine \n\n";} + +print STDERR " +WARNING: Your phrase tables must be sorted (with LC_ALL=C) !! +****************************** +Combination mode is [$combination_mode] +****************************** +"; + +my @tables = @ARGV; +my $nbtables = scalar(@tables); + +########################################### + +# The newSourceMaxLength option requires reading all the first PT before starting the combination +my %sourcePhrasesPT1; +if($combination_mode eq "fillup" && $newSourceMaxLength>-1) { + my $table1=$tables[0]; + $table1 =~ s/(.*\.gz)\s*$/gzip -dc < $1|/; + open(TABLE1, "$table1") or die "Cannot open $table1: ($!)\n"; + while(my $line=) { + $line=~m/^(.*?)$delim_RE/; + $sourcePhrasesPT1{$1}++; + } + close(TABLE1); +} + +my @table_files=(); +foreach my $table (@tables) { + $table =~ s/(.*\.gz)\s*$/gzip -dc < $1|/; + #localize the file glob, so FILE is unique to the inner loop. + local *FILE; + open(FILE, "$table") or die "Cannot open $table: ($!)\n"; + push(@table_files, *FILE); +} + + +# Read first line from all tables to find number of weights (and sanity checks) +my @read_ppairs=(); +my $nbscores = &read_line_from_tables(\@table_files, \@read_ppairs); +print STDERR "Each phrase table contains $nbscores features.\n"; + +########################################### + +if($phPenalty_idx!=-1) { + if($phPenalty_idx<0 || $phPenalty_idx>=$nbscores) { + die "Invalid value for option phpenalty-at! Should be in the range [0,($nbscores-1)]\n\n"; + } + else { print STDERR "Phrase penalty at index $phPenalty_idx\n"; } +} + +#if($weights_str ne "") { die "Weights option NOT supported yet. Can only use uniform (1/nbscores)\n\n"; } +#my $unifw = 1/$nbtables; + +my @weights=(); # Array of arrays each containing the feature weights for a phrase table +if($combination_mode eq "interp") { + my @table_level_weights=(); + if($weights_str eq "") { + @table_level_weights= ((1/$nbtables) x $nbtables); # assuming uniform weights + } + else { + @table_level_weights= split(/,/, $weights_str); + if(scalar(@table_level_weights) != $nbtables) { + die "$usage\n Invalid string for option --weights! Must be a comma-separated list of floats, one per ph.table.\n"; + } + } + + for(my $i=0; $i<$nbtables; $i++) { + my @weights_pt = (($table_level_weights[$i]) x $nbscores); + if($phPenalty_idx!=-1) { + $weights_pt[$phPenalty_idx]=0; + } + print STDERR "WEIGHTS-PT_$i: ", join(" -- ", @weights_pt), "\n"; + $weights[$i] = \@weights_pt; + } + print STDERR "EPSILON: $epsilon \n"; +} + + +########################################### + +my @empty_ppair=(""); +my @epsilons = (($epsilon) x $nbscores); +if($phPenalty_idx>-1) { + pop @epsilons; +} + +my $nbPpairs_inAll=0; +my @nbPairs_found_only_in=((0) x $nbtables); +my $MINSCORE=1; + +print STDERR "Working...\n\n"; +while(1) { + my $min_ppair=""; + my $reached_end_of_tables=1; + my @tablesContainingPpair=((0) x $nbtables); + for(my $i=0; $i<$nbtables; $i++) { + my $ppair=$read_ppairs[$i]->[0]; + if($ppair ne "") { + $reached_end_of_tables=0; + if($min_ppair eq "" || $ppair lt $min_ppair) { + $min_ppair=$ppair; + @tablesContainingPpair=((0) x $nbtables); + $tablesContainingPpair[$i]=1; + } + elsif($ppair eq $min_ppair) { + $tablesContainingPpair[$i]=1; + } + } + } + last if($reached_end_of_tables); + + ## Actual combination is performed here: + &combine_ppair(\@read_ppairs, \@tablesContainingPpair); + + &read_line_from_tables(\@table_files, \@read_ppairs, \@tablesContainingPpair); + +} + +print STDERR "...done!\n"; + +print STDERR "The minimum score in all tables is $MINSCORE\n"; + +if($combination_mode eq "stats") { +my $tot_ppairs=0; +print " +# entries +found in all tables: $nbPpairs_inAll\n"; + +for(my $i=0; $i<$nbtables; $i++) { + print "found only in PT_$i: $nbPairs_found_only_in[$i]\n"; +} + +} + +#################################### +sub combine_ppair(PPAIRS_REFARRAY, TABLE_INDICES_REFARRAY) { + my $ra_ppairs=shift; # 1st item: phrase-pair key (string); + # 2nd item: ref.array of scores; + # 3rd item: additional info (string, may be empty) + + my $ra_toRead=shift; # Important: this says which phrase tables contain the ph.pair currently processed + + my $ppair=""; + my @scores=(); + my $additional_info=""; + + my $to_print=1; + + if($debug) { + print STDERR "combine_ppair:\n"; + for(my $i=0; $i<$nbtables; $i++) { + if($ra_toRead->[$i]) { + print STDERR "ppair_$i= ", join (" // ", @{$ra_ppairs->[$i]}), "\n"; + } + } + } + + if($combination_mode eq "stats") { + $to_print=0; + my $found_in=-1; + my $nb_found=0; + for(my $i=0; $i<$nbtables; $i++) { + if($ra_toRead->[$i]) { + $found_in=$i; + $nb_found++; + } + } + if($nb_found==1) { $nbPairs_found_only_in[$found_in]++; } + elsif($nb_found==$nbtables) { $nbPpairs_inAll++; } + } + ### Fill-up + additional binary feature + elsif($combination_mode eq "fillup") { + my @bin_feats=(($exp_zero) x ($nbtables-1)); + for(my $i=0; $i<$nbtables; $i++) { + if($ra_toRead->[$i]) { + $ppair= shift(@{$ra_ppairs->[$i]}); + # pruning criteria are applied here: + if($i>0 && $newSourceMaxLength>-1) { + $ppair=~m/^(.*?)$delim_RE/; + if(scalar(split(/ +/, $1)) > $newSourceMaxLength && + !defined($sourcePhrasesPT1{$1})) + { $to_print=0; } + } +# @scores= @{$ra_ppairs->[$i]}; + @scores = @{shift(@{$ra_ppairs->[$i]})}; + # binary feature for ph.pair provenance fires here + if($i>0) { $bin_feats[$i-1]=$exp_one; } + $additional_info=shift(@{$ra_ppairs->[$i]}); + last; + } + } + push(@scores, @bin_feats); + } + ### Linear interpolation + elsif($combination_mode eq "interp") { + my $firstPpair=-1; + @scores=((0) x $nbscores); + for(my $i=0; $i<$nbtables; $i++) { + if($ra_toRead->[$i]) { + if($firstPpair==-1) { $firstPpair=$i; } + $ppair= shift(@{$ra_ppairs->[$i]}); + my @scoresPT = @{shift(@{$ra_ppairs->[$i]})}; + for(my $j=0; $j<$nbscores; $j++) { +# $scores[$j]+= $weights[$i]->[$j]* $ra_ppairs->[$i][$j]; + $scores[$j]+= $weights[$i]->[$j]* $scoresPT[$j]; + } + } + else { + for(my $j=0; $j<$nbscores; $j++) { + $scores[$j]+= $weights[$i]->[$j]* $epsilon; + } + } + if($phPenalty_idx!=-1) { + $scores[$phPenalty_idx]= $phPenalty; + } + } + if($debug) { print STDERR "..taking info from ptable_$firstPpair\n"; } + $additional_info= shift(@{$ra_ppairs->[$firstPpair]}); + } + ### Union + feature concatenation + elsif($combination_mode eq "union") { + my $firstPpair=-1; + for(my $i=0; $i<$nbtables; $i++) { + if($ra_toRead->[$i]) { + if($firstPpair==-1) { $firstPpair=$i; } + $ppair= shift(@{$ra_ppairs->[$i]}); + my @scoresPT= @{shift(@{$ra_ppairs->[$i]})}; + if($phPenalty_idx!=-1) { +# splice(@{$ra_ppairs->[$i]}, $phPenalty_idx, 1); + splice(@scoresPT, $phPenalty_idx, 1); + } +# push(@scores, @{$ra_ppairs->[$i]}); + push(@scores, @scoresPT); + } + else { + push(@scores, @epsilons); + } + } + if($phPenalty_idx!=-1) { + push(@scores, $phPenalty); + } + if($debug) { print STDERR "..taking info from ptable_$firstPpair\n"; } + $additional_info= shift(@{$ra_ppairs->[$firstPpair]}); + } + ### Intersect + features from first table + elsif($combination_mode eq "intersect1") { + $to_print=0; + my $found_in_all=1; + for(my $i=0; $i<$nbtables; $i++) { + if(!$ra_toRead->[$i]) { + $found_in_all=0; + last; + } + } + if($found_in_all) { + $to_print=1; + $ppair= shift(@{$ra_ppairs->[0]}); +# @scores= @{$ra_ppairs->[0]}; + @scores= @{shift(@{$ra_ppairs->[0]})}; + $additional_info= shift(@{$ra_ppairs->[0]}); + } + } + else { + die "$usage\nUnknown combination mode!\n"; + } + + + if($to_print) { + if($additional_info eq "") { + print $ppair, join(" ", @scores), "\n"; + }else { + print $ppair, join(" ", @scores), $delim, $additional_info, "\n"; + } + } +} + +#################################### +# Read lines from all filehandles given in FILES_REFARRAY, +# or from the files whose indices are assigned 1 in the array TABLE_INDICES_REFARRAY +# Parse each of them as a phrase pair entry and stores it to the corresponding position of PPAIRS_REFARRAY +sub read_line_from_tables(FILES_REFARRAY, PPAIRS_REFARRAY, TABLE_INDICES_REFARRAY) { + my $ra_files=shift; + my $ra_ppairs=shift; + + my $ra_toRead=shift; + my @toRead=((1) x $nbtables); # by default read from all files + if($ra_toRead ne "") { + @toRead=@$ra_toRead; + } + + my $nbscores=-1; + my $key=""; my $additional_info=""; + for(my $i=0; $i<$nbtables; $i++) { + next if($toRead[$i]==0); + my @ppair=(); + my $file=$ra_files->[$i]; + if(my $line = <$file>) { + chomp $line; + my @fields = split(/$delim_RE/, $line); + if(scalar(@fields)<3) { + die "Invalid phrase table entry:\n$line\n"; + } + my @scores = split(/\s+/, $fields[2]); + foreach my $score (@scores) { + if($score<$MINSCORE) { $MINSCORE=$score; } + } + # Get nb of scores from the 1st table. Check that all tables provide the same nb of scores, + # unless mode is 'intersect' (then it doesn't matter as scores are taken only from 1st table) + if($nbscores==-1) { + $nbscores=scalar(@scores); + } elsif($nbscores!=scalar(@scores) && $combination_mode ne "intersect1") { + die "Wrong number of scores in table-$i! Should be $nbscores\n"; + } + # Get additional fields if any (word aligment, phrase counts etc.) + if(scalar(@fields)>3) { + $additional_info=join($delim, splice(@fields,3)); + #print STDOUT "additional_info:__{$additional_info}__\n"; + } + my $key = "$fields[0]$delim$fields[1]$delim"; ## IMPORTANT: the | delimiter at the end of the phrase pair is crucial to preserve sorting!! + push(@ppair, $key, \@scores, $additional_info); + } + else { + push(@ppair, ""); + } + $ra_ppairs->[$i]=\@ppair; + } + + return $nbscores; +} + +######### +} + + +&main; diff --git a/contrib/fuzzy-match/Makefile b/contrib/fuzzy-match/Makefile new file mode 100644 index 000000000..5bb884a51 --- /dev/null +++ b/contrib/fuzzy-match/Makefile @@ -0,0 +1,16 @@ +all: suffix-test fuzzy-match fuzzy-match2 + +clean: + rm -f *.o + +.cpp.o: + g++ -O6 -g -c $< + +suffix-test: Vocabulary.o SuffixArray.o suffix-test.o + g++ Vocabulary.o SuffixArray.o suffix-test.o -o suffix-test + +fuzzy-match: Vocabulary.o SuffixArray.o old/fuzzy-match.o + g++ Vocabulary.o SuffixArray.o fuzzy-match.o -o fuzzy-match + +fuzzy-match2: Vocabulary.o SuffixArray.o fuzzy-match2.o Util.o + g++ Vocabulary.o SuffixArray.o fuzzy-match2.o Util.o -o fuzzy-match2 diff --git a/contrib/fuzzy-match/Match.h b/contrib/fuzzy-match/Match.h new file mode 100644 index 000000000..6fc8bb42f --- /dev/null +++ b/contrib/fuzzy-match/Match.h @@ -0,0 +1,29 @@ +// +// Match.h +// fuzzy-match +// +// Created by Hieu Hoang on 25/07/2012. +// Copyright 2012 __MyCompanyName__. All rights reserved. +// + +#ifndef fuzzy_match_Match_h +#define fuzzy_match_Match_h + +/* data structure for n-gram match between input and corpus */ + +class Match { +public: + int input_start; + int input_end; + int tm_start; + int tm_end; + int min_cost; + int max_cost; + int internal_cost; + Match( int is, int ie, int ts, int te, int min, int max, int i ) + :input_start(is), input_end(ie), tm_start(ts), tm_end(te), min_cost(min), max_cost(max), internal_cost(i) + {} +}; + + +#endif diff --git a/contrib/fuzzy-match/SentenceAlignment.h b/contrib/fuzzy-match/SentenceAlignment.h new file mode 100644 index 000000000..4d92fd635 --- /dev/null +++ b/contrib/fuzzy-match/SentenceAlignment.h @@ -0,0 +1,48 @@ +// +// SentenceAlignment.h +// fuzzy-match +// +// Created by Hieu Hoang on 25/07/2012. +// Copyright 2012 __MyCompanyName__. All rights reserved. +// + +#ifndef fuzzy_match_SentenceAlignment_h +#define fuzzy_match_SentenceAlignment_h + +#include +#include "Vocabulary.h" + +extern Vocabulary vocabulary; + +struct SentenceAlignment +{ + int count; + vector< WORD_ID > target; + vector< pair > alignment; + + SentenceAlignment() + {} + + string getTargetString() const + { + stringstream strme; + for (size_t i = 0; i < target.size(); ++i) { + const WORD &word = vocabulary.GetWord(target[i]); + strme << word << " "; + } + return strme.str(); + } + + string getAlignmentString() const + { + stringstream strme; + for (size_t i = 0; i < alignment.size(); ++i) { + const pair &alignPair = alignment[i]; + strme << alignPair.first << "-" << alignPair.second << " "; + } + return strme.str(); + } + +}; + +#endif diff --git a/contrib/fuzzy-match/SuffixArray.cpp b/contrib/fuzzy-match/SuffixArray.cpp new file mode 100644 index 000000000..e0aa3da91 --- /dev/null +++ b/contrib/fuzzy-match/SuffixArray.cpp @@ -0,0 +1,244 @@ +#include "SuffixArray.h" +#include +#include +#include + +using namespace std; + +SuffixArray::SuffixArray( string fileName ) +{ + m_vcb.StoreIfNew( "" ); + m_endOfSentence = m_vcb.StoreIfNew( "" ); + + ifstream extractFile; + char line[LINE_MAX_LENGTH]; + + // count the number of words first; + extractFile.open(fileName.c_str()); + istream *fileP = &extractFile; + m_size = 0; + size_t sentenceCount = 0; + while(!fileP->eof()) { + SAFE_GETLINE((*fileP), line, LINE_MAX_LENGTH, '\n'); + if (fileP->eof()) break; + vector< WORD_ID > words = m_vcb.Tokenize( line ); + m_size += words.size() + 1; + sentenceCount++; + } + extractFile.close(); + cerr << m_size << " words (incl. sentence boundaries)" << endl; + + // allocate memory + m_array = (WORD_ID*) calloc( sizeof( WORD_ID ), m_size ); + m_index = (INDEX*) calloc( sizeof( INDEX ), m_size ); + m_wordInSentence = (char*) calloc( sizeof( char ), m_size ); + m_sentence = (size_t*) calloc( sizeof( size_t ), m_size ); + m_sentenceLength = (char*) calloc( sizeof( char ), sentenceCount ); + + // fill the array + int wordIndex = 0; + int sentenceId = 0; + extractFile.open(fileName.c_str()); + fileP = &extractFile; + while(!fileP->eof()) { + SAFE_GETLINE((*fileP), line, LINE_MAX_LENGTH, '\n'); + if (fileP->eof()) break; + vector< WORD_ID > words = m_vcb.Tokenize( line ); + vector< WORD_ID >::const_iterator i; + + for( i=words.begin(); i!=words.end(); i++) + { + m_index[ wordIndex ] = wordIndex; + m_sentence[ wordIndex ] = sentenceId; + m_wordInSentence[ wordIndex ] = i-words.begin(); + m_array[ wordIndex++ ] = *i; + } + m_index[ wordIndex ] = wordIndex; + m_array[ wordIndex++ ] = m_endOfSentence; + m_sentenceLength[ sentenceId++ ] = words.size(); + } + extractFile.close(); + cerr << "done reading " << wordIndex << " words, " << sentenceId << " sentences." << endl; + // List(0,9); + + // sort + m_buffer = (INDEX*) calloc( sizeof( INDEX ), m_size ); + Sort( 0, m_size-1 ); + free( m_buffer ); + cerr << "done sorting" << endl; +} + +// good ol' quick sort +void SuffixArray::Sort(INDEX start, INDEX end) { + if (start == end) return; + INDEX mid = (start+end+1)/2; + Sort( start, mid-1 ); + Sort( mid, end ); + + // merge + int i = start; + int j = mid; + int k = 0; + int length = end-start+1; + while( k end ) + { + m_buffer[ k++ ] = m_index[ i++ ]; + } + else { + if (CompareIndex( m_index[i], m_index[j] ) < 0) + { + m_buffer[ k++ ] = m_index[ i++ ]; + } + else + { + m_buffer[ k++ ] = m_index[ j++ ]; + } + } + } + + memcpy( ((char*)m_index) + sizeof( INDEX ) * start, + ((char*)m_buffer), sizeof( INDEX ) * (end-start+1) ); +} + +SuffixArray::~SuffixArray() +{ + free(m_index); + free(m_array); +} + +int SuffixArray::CompareIndex( INDEX a, INDEX b ) const +{ + // skip over identical words + INDEX offset = 0; + while( a+offset < m_size && + b+offset < m_size && + m_array[ a+offset ] == m_array[ b+offset ] ) + { offset++; } + + if( a+offset == m_size ) return -1; + if( b+offset == m_size ) return 1; + return CompareWord( m_array[ a+offset ], m_array[ b+offset ] ); +} + +inline int SuffixArray::CompareWord( WORD_ID a, WORD_ID b ) const +{ + // cerr << "c(" << m_vcb.GetWord(a) << ":" << m_vcb.GetWord(b) << ")=" << m_vcb.GetWord(a).compare( m_vcb.GetWord(b) ) << endl; + return m_vcb.GetWord(a).compare( m_vcb.GetWord(b) ); +} + +int SuffixArray::Count( const vector< WORD > &phrase ) +{ + INDEX dummy; + return LimitedCount( phrase, m_size, dummy, dummy, 0, m_size-1 ); +} + +bool SuffixArray::MinCount( const vector< WORD > &phrase, INDEX min ) +{ + INDEX dummy; + return LimitedCount( phrase, min, dummy, dummy, 0, m_size-1 ) >= min; +} + +bool SuffixArray::Exists( const vector< WORD > &phrase ) +{ + INDEX dummy; + return LimitedCount( phrase, 1, dummy, dummy, 0, m_size-1 ) == 1; +} + +int SuffixArray::FindMatches( const vector< WORD > &phrase, INDEX &firstMatch, INDEX &lastMatch, INDEX search_start, INDEX search_end ) +{ + return LimitedCount( phrase, m_size, firstMatch, lastMatch, search_start, search_end ); +} + +int SuffixArray::LimitedCount( const vector< WORD > &phrase, INDEX min, INDEX &firstMatch, INDEX &lastMatch, INDEX search_start, INDEX search_end ) +{ + // cerr << "FindFirst\n"; + INDEX start = search_start; + INDEX end = (search_end == -1) ? (m_size-1) : search_end; + INDEX mid = FindFirst( phrase, start, end ); + // cerr << "done\n"; + if (mid == m_size) return 0; // no matches + if (min == 1) return 1; // only existance check + + int matchCount = 1; + + //cerr << "before...\n"; + firstMatch = FindLast( phrase, mid, start, -1 ); + matchCount += mid - firstMatch; + + //cerr << "after...\n"; + lastMatch = FindLast( phrase, mid, end, 1 ); + matchCount += lastMatch - mid; + + return matchCount; +} + +SuffixArray::INDEX SuffixArray::FindLast( const vector< WORD > &phrase, INDEX start, INDEX end, int direction ) +{ + end += direction; + while(true) + { + INDEX mid = ( start + end + (direction>0 ? 0 : 1) )/2; + + int match = Match( phrase, mid ); + int matchNext = Match( phrase, mid+direction ); + //cerr << "\t" << start << ";" << mid << ";" << end << " -> " << match << "," << matchNext << endl; + + if (match == 0 && matchNext != 0) return mid; + + if (match == 0) // mid point is a match + start = mid; + else + end = mid; + } +} + +SuffixArray::INDEX SuffixArray::FindFirst( const vector< WORD > &phrase, INDEX &start, INDEX &end ) +{ + while(true) + { + INDEX mid = ( start + end + 1 )/2; + //cerr << "FindFirst(" << start << ";" << mid << ";" << end << ")\n"; + int match = Match( phrase, mid ); + + if (match == 0) return mid; + if (start >= end && match != 0 ) return m_size; + + if (match > 0) + start = mid+1; + else + end = mid-1; + } +} + +int SuffixArray::Match( const vector< WORD > &phrase, INDEX index ) +{ + INDEX pos = m_index[ index ]; + for(INDEX i=0; i &phrase ); + bool MinCount( const vector< WORD > &phrase, INDEX min ); + bool Exists( const vector< WORD > &phrase ); + int FindMatches( const vector< WORD > &phrase, INDEX &firstMatch, INDEX &lastMatch, INDEX search_start = 0, INDEX search_end = -1 ); + int LimitedCount( const vector< WORD > &phrase, INDEX min, INDEX &firstMatch, INDEX &lastMatch, INDEX search_start = -1, INDEX search_end = 0 ); + INDEX FindFirst( const vector< WORD > &phrase, INDEX &start, INDEX &end ); + INDEX FindLast( const vector< WORD > &phrase, INDEX start, INDEX end, int direction ); + int Match( const vector< WORD > &phrase, INDEX index ); + void List( INDEX start, INDEX end ); + inline INDEX GetPosition( INDEX index ) { return m_index[ index ]; } + inline size_t GetSentence( INDEX position ) { return m_sentence[position]; } + inline char GetWordInSentence( INDEX position ) { return m_wordInSentence[position]; } + inline char GetSentenceLength( size_t sentenceId ) { return m_sentenceLength[sentenceId]; } + inline INDEX GetSize() { return m_size; } +}; diff --git a/contrib/fuzzy-match/Util.cpp b/contrib/fuzzy-match/Util.cpp new file mode 100644 index 000000000..4d750791e --- /dev/null +++ b/contrib/fuzzy-match/Util.cpp @@ -0,0 +1,147 @@ +// +// Util.cpp +// fuzzy-match +// +// Created by Hieu Hoang on 26/07/2012. +// Copyright 2012 __MyCompanyName__. All rights reserved. +// + +#include +#include +#include "Util.h" +#include "SentenceAlignment.h" +#include "SuffixArray.h" + +void load_corpus( const char* fileName, vector< vector< WORD_ID > > &corpus ) +{ // source + ifstream fileStream; + fileStream.open(fileName); + if (!fileStream) { + cerr << "file not found: " << fileName << endl; + exit(1); + } + cerr << "loading " << fileName << endl; + + istream *fileStreamP = &fileStream; + + char line[LINE_MAX_LENGTH]; + while(true) + { + SAFE_GETLINE((*fileStreamP), line, LINE_MAX_LENGTH, '\n'); + if (fileStreamP->eof()) break; + corpus.push_back( vocabulary.Tokenize( line ) ); + } +} + +void load_target( const char* fileName, vector< vector< SentenceAlignment > > &corpus) +{ + ifstream fileStream; + fileStream.open(fileName); + if (!fileStream) { + cerr << "file not found: " << fileName << endl; + exit(1); + } + cerr << "loading " << fileName << endl; + + istream *fileStreamP = &fileStream; + + WORD_ID delimiter = vocabulary.StoreIfNew("|||"); + + int lineNum = 0; + char line[LINE_MAX_LENGTH]; + while(true) + { + SAFE_GETLINE((*fileStreamP), line, LINE_MAX_LENGTH, '\n'); + if (fileStreamP->eof()) break; + + vector toks = vocabulary.Tokenize( line ); + + corpus.push_back(vector< SentenceAlignment >()); + vector< SentenceAlignment > &vec = corpus.back(); + + vec.push_back(SentenceAlignment()); + SentenceAlignment *sentence = &vec.back(); + + const WORD &countStr = vocabulary.GetWord(toks[0]); + sentence->count = atoi(countStr.c_str()); + + for (size_t i = 1; i < toks.size(); ++i) { + WORD_ID wordId = toks[i]; + + if (wordId == delimiter) { + // target and alignments can have multiple sentences. + vec.push_back(SentenceAlignment()); + sentence = &vec.back(); + + // count + ++i; + + const WORD &countStr = vocabulary.GetWord(toks[i]); + sentence->count = atoi(countStr.c_str()); + } + else { + // just a normal word, add + sentence->target.push_back(wordId); + } + } + + ++lineNum; + + } + +} + + +void load_alignment( const char* fileName, vector< vector< SentenceAlignment > > &corpus ) +{ + ifstream fileStream; + fileStream.open(fileName); + if (!fileStream) { + cerr << "file not found: " << fileName << endl; + exit(1); + } + cerr << "loading " << fileName << endl; + + istream *fileStreamP = &fileStream; + + string delimiter = "|||"; + + int lineNum = 0; + char line[LINE_MAX_LENGTH]; + while(true) + { + SAFE_GETLINE((*fileStreamP), line, LINE_MAX_LENGTH, '\n'); + if (fileStreamP->eof()) break; + + vector< SentenceAlignment > &vec = corpus[lineNum]; + size_t targetInd = 0; + SentenceAlignment *sentence = &vec[targetInd]; + + vector toks = Tokenize(line); + + for (size_t i = 0; i < toks.size(); ++i) { + string &tok = toks[i]; + + if (tok == delimiter) { + // target and alignments can have multiple sentences. + ++targetInd; + sentence = &vec[targetInd]; + + ++i; + } + else { + // just a normal alignment, add + vector alignPoint = Tokenize(tok, "-"); + assert(alignPoint.size() == 2); + sentence->alignment.push_back(pair(alignPoint[0], alignPoint[1])); + } + } + + ++lineNum; + + } +} + + + + diff --git a/contrib/fuzzy-match/Util.h b/contrib/fuzzy-match/Util.h new file mode 100644 index 000000000..7bb13d032 --- /dev/null +++ b/contrib/fuzzy-match/Util.h @@ -0,0 +1,87 @@ +// +// Util.h +// fuzzy-match +// +// Created by Hieu Hoang on 25/07/2012. +// Copyright 2012 __MyCompanyName__. All rights reserved. +// + +#ifndef fuzzy_match_Util_h +#define fuzzy_match_Util_h + +#include +#include +#include "Vocabulary.h" + +class SentenceAlignment; + +void load_corpus( const char* fileName, std::vector< std::vector< WORD_ID > > &corpus ); +void load_target( const char* fileName, std::vector< std::vector< SentenceAlignment > > &corpus); +void load_alignment( const char* fileName, std::vector< std::vector< SentenceAlignment > > &corpus ); + +/** + * Convert vector of type T to string + */ +template +std::string Join(const std::string& delimiter, const std::vector& items) +{ + std::ostringstream outstr; + if(items.size() == 0) return ""; + outstr << items[0]; + for(unsigned int i = 1; i < items.size(); i++) + outstr << delimiter << items[i]; + return outstr.str(); +} + +//! convert string to variable of type T. Used to reading floats, int etc from files +template +inline T Scan(const std::string &input) +{ + std::stringstream stream(input); + T ret; + stream >> ret; + return ret; +} + +//! convert vectors of string to vectors of type T variables +template +inline std::vector Scan(const std::vector< std::string > &input) +{ + std::vector output(input.size()); + for (size_t i = 0 ; i < input.size() ; i++) { + output[i] = Scan( input[i] ); + } + return output; +} + +inline std::vector Tokenize(const std::string& str, + const std::string& delimiters = " \t") +{ + std::vector tokens; + // Skip delimiters at beginning. + std::string::size_type lastPos = str.find_first_not_of(delimiters, 0); + // Find first "non-delimiter". + std::string::size_type pos = str.find_first_of(delimiters, lastPos); + + while (std::string::npos != pos || std::string::npos != lastPos) { + // Found a token, add it to the vector. + tokens.push_back(str.substr(lastPos, pos - lastPos)); + // Skip delimiters. Note the "not_of" + lastPos = str.find_first_not_of(delimiters, pos); + // Find next "non-delimiter" + pos = str.find_first_of(delimiters, lastPos); + } + + return tokens; +} + +template +inline std::vector Tokenize( const std::string &input + , const std::string& delimiters = " \t") +{ + std::vector stringVector = Tokenize(input, delimiters); + return Scan( stringVector ); +} + + +#endif diff --git a/contrib/fuzzy-match/Vocabulary.cpp b/contrib/fuzzy-match/Vocabulary.cpp new file mode 100644 index 000000000..4492eec95 --- /dev/null +++ b/contrib/fuzzy-match/Vocabulary.cpp @@ -0,0 +1,45 @@ +// $Id: Vocabulary.cpp 1565 2008-02-22 14:42:01Z bojar $ +#include "Vocabulary.h" + +// as in beamdecoder/tables.cpp +vector Vocabulary::Tokenize( const char input[] ) { + vector< WORD_ID > token; + bool betweenWords = true; + int start=0; + int i=0; + for(; input[i] != '\0'; i++) { + bool isSpace = (input[i] == ' ' || input[i] == '\t'); + + if (!isSpace && betweenWords) { + start = i; + betweenWords = false; + } + else if (isSpace && !betweenWords) { + token.push_back( StoreIfNew ( string( input+start, i-start ) ) ); + betweenWords = true; + } + } + if (!betweenWords) + token.push_back( StoreIfNew ( string( input+start, i-start ) ) ); + return token; +} + +WORD_ID Vocabulary::StoreIfNew( const WORD& word ) { + map::iterator i = lookup.find( word ); + + if( i != lookup.end() ) + return i->second; + + WORD_ID id = vocab.size(); + vocab.push_back( word ); + lookup[ word ] = id; + return id; +} + +WORD_ID Vocabulary::GetWordID( const WORD &word ) { + map::iterator i = lookup.find( word ); + if( i == lookup.end() ) + return 0; + WORD_ID w= (WORD_ID) i->second; + return w; +} diff --git a/contrib/fuzzy-match/Vocabulary.h b/contrib/fuzzy-match/Vocabulary.h new file mode 100644 index 000000000..3e48847a7 --- /dev/null +++ b/contrib/fuzzy-match/Vocabulary.h @@ -0,0 +1,40 @@ +// $Id: tables-core.h 1470 2007-10-02 21:43:54Z redpony $ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#define MAX_LENGTH 10000 + +#define SAFE_GETLINE(_IS, _LINE, _SIZE, _DELIM) { \ + _IS.getline(_LINE, _SIZE, _DELIM); \ + if(_IS.fail() && !_IS.bad() && !_IS.eof()) _IS.clear(); \ + if (_IS.gcount() == _SIZE-1) { \ + cerr << "Line too long! Buffer overflow. Delete lines >=" \ + << _SIZE << " chars or raise MAX_LENGTH in phrase-extract/tables-core.cpp" \ + << endl; \ + exit(1); \ + } \ + } + +typedef string WORD; +typedef unsigned int WORD_ID; + +class Vocabulary { + public: + map lookup; + vector< WORD > vocab; + WORD_ID StoreIfNew( const WORD& ); + WORD_ID GetWordID( const WORD& ); + vector Tokenize( const char[] ); + inline WORD &GetWord( WORD_ID id ) const { WORD &i = (WORD&) vocab[ id ]; return i; } +}; diff --git a/contrib/fuzzy-match/fuzzy-match2.cpp b/contrib/fuzzy-match/fuzzy-match2.cpp new file mode 100644 index 000000000..c1252aa03 --- /dev/null +++ b/contrib/fuzzy-match/fuzzy-match2.cpp @@ -0,0 +1,460 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "SentenceAlignment.h" +#include "fuzzy-match2.h" +#include "SuffixArray.h" + +/** This implementation is explained in + Koehn and Senellart: "Fast Approximate String Matching + with Suffix Arrays and A* Parsing" (AMTA 2010) ***/ + +using namespace std; + +int main(int argc, char* argv[]) +{ + vector< vector< WORD_ID > > source, input; + vector< vector< SentenceAlignment > > targetAndAlignment; + + + while(1) { + static struct option long_options[] = { + {"basic", no_argument, &basic_flag, 1}, + {"word", no_argument, &lsed_flag, 0}, + {"unrefined", no_argument, &refined_flag, 0}, + {"nolengthfilter", no_argument, &length_filter_flag, 0}, + {"noparse", no_argument, &parse_flag, 0}, + {"multiple", no_argument, &multiple_flag, 1}, + {"minmatch", required_argument, 0, 'm'}, + {0, 0, 0, 0} + }; + int option_index = 0; + int c = getopt_long (argc, argv, "m:", long_options, &option_index); + if (c == -1) break; + switch (c) { + case 0: +// if (long_options[option_index].flag != 0) +// break; +// printf ("option %s", long_options[option_index].name); +// if (optarg) +// printf (" with arg %s", optarg); +// printf ("\n"); + break; + case 'm': + min_match = atoi(optarg); + if (min_match < 1 || min_match > 100) { + cerr << "error: --minmatch must have value in range 1..100\n"; + exit(1); + } + cerr << "setting min match to " << min_match << endl; + break; + default: + cerr << "usage: syntax: ./fuzzy-match input corpus [--basic] [--word] [--minmatch 1..100]\n"; + exit(1); + } + } + if (lsed_flag) { cerr << "lsed\n"; } + if (basic_flag) { cerr << "basic\n"; } + if (refined_flag) { cerr << "refined\n"; } + if (length_filter_flag) { cerr << "length filter\n"; } + if (parse_flag) { cerr << "parse\n"; } +// exit(1); + + + if (optind+4 != argc) { + cerr << "syntax: ./fuzzy-match input source target alignment [--basic] [--word] [--minmatch 1..100]\n"; + exit(1); + } + + load_corpus(argv[optind], input); + load_corpus(argv[optind+1], source); + load_target(argv[optind+2], targetAndAlignment); + load_alignment(argv[optind+3], targetAndAlignment); + + // ./fuzzy-match input corpus [-basic] + +// load_corpus("../corpus/tm.truecased.4.en", source); +// load_corpus("../corpus/tm.truecased.4.it", target); +// load_corpus("../evaluation/test.input.tc.4", input); + +// load_corpus("../../acquis-truecase/corpus/acquis.truecased.190.en", source); +// load_corpus("../../acquis-truecase/evaluation/ac-test.input.tc.190", input); + +// load_corpus("../corpus/tm.truecased.16.en", source); +// load_corpus("../evaluation/test.input.tc.16", input); + + if (basic_flag) { + cerr << "using basic method\n"; + clock_t start_main_clock2 = clock(); + basic_fuzzy_match( source, input ); + cerr << "total: " << (1000 * (clock()-start_main_clock2) / CLOCKS_PER_SEC) << endl; + exit(1); + } + + cerr << "number of input sentences " << input.size() << endl; + + cerr << "creating suffix array...\n"; +// SuffixArray suffixArray( "../corpus/tm.truecased.4.en" ); +// SuffixArray suffixArray( "../../acquis-truecase/corpus/acquis.truecased.190.en" ); + SuffixArray suffixArray( argv[optind+1] ); + + clock_t start_main_clock = clock(); + + // looping through all input sentences... + cerr << "looping...\n"; + for(unsigned int sentenceInd = 0; sentenceInd < input.size(); sentenceInd++) + { + clock_t start_clock = clock(); + // if (i % 10 == 0) cerr << "."; + + // establish some basic statistics + + // int input_length = compute_length( input[i] ); + int input_length = input[sentenceInd].size(); + int best_cost = input_length * (100-min_match) / 100 + 1; + + int match_count = 0; // how many substring matches to be considered + //cerr << endl << "sentence " << i << ", length " << input_length << ", best_cost " << best_cost << endl; + + // find match ranges in suffix array + vector< vector< pair< SuffixArray::INDEX, SuffixArray::INDEX > > > match_range; + for(size_t start=0;start substring; + bool stillMatched = true; + vector< pair< SuffixArray::INDEX, SuffixArray::INDEX > > matchedAtThisStart; + //cerr << "start: " << start; + for(int word=start; stillMatched && word > sentence_match; + map< int, int > sentence_match_word_count; + + // go through all matches, longest first + for(int length = input[sentenceInd].size(); length >= 1; length--) + { + // do not create matches, if these are handled by the short match function + if (length <= short_match_max_length( input_length ) ) + { + continue; + } + + unsigned int count = 0; + for(int start = 0; start <= input[sentenceInd].size() - length; start++) + { + if (match_range[start].size() >= length) + { + pair< SuffixArray::INDEX, SuffixArray::INDEX > &range = match_range[start][length-1]; + // cerr << " (" << range.first << "," << range.second << ")"; + count += range.second - range.first + 1; + + for(SuffixArray::INDEX i=range.first; i<=range.second; i++) + { + int position = suffixArray.GetPosition( i ); + + // sentence length mismatch + size_t sentence_id = suffixArray.GetSentence( position ); + int sentence_length = suffixArray.GetSentenceLength( sentence_id ); + int diff = abs( (int)sentence_length - (int)input_length ); + // cerr << endl << i << "\tsentence " << sentence_id << ", length " << sentence_length; + //if (length <= 2 && input_length>=5 && + // sentence_match.find( sentence_id ) == sentence_match.end()) + // continue; + + if (diff > best_cost) + continue; + + // compute minimal cost + int start_pos = suffixArray.GetWordInSentence( position ); + int end_pos = start_pos + length-1; + // cerr << endl << "\t" << start_pos << "-" << end_pos << " (" << sentence_length << ") vs. " + // << start << "-" << (start+length-1) << " (" << input_length << ")"; + // different number of prior words -> cost is at least diff + int min_cost = abs( start - start_pos ); + + // same number of words, but not sent. start -> cost is at least 1 + if (start == start_pos && start>0) + min_cost++; + + // different number of remaining words -> cost is at least diff + min_cost += abs( ( sentence_length-1 - end_pos ) - + ( input_length-1 - (start+length-1) ) ); + + // same number of words, but not sent. end -> cost is at least 1 + if ( sentence_length-1 - end_pos == + input_length-1 - (start+length-1) + && end_pos != sentence_length-1 ) + min_cost++; + + // cerr << " -> min_cost " << min_cost; + if (min_cost > best_cost) + continue; + + // valid match + match_count++; + + // compute maximal cost + int max_cost = max( start, start_pos ) + + max( sentence_length-1 - end_pos, + input_length-1 - (start+length-1) ); + // cerr << ", max_cost " << max_cost; + + Match m = Match( start, start+length-1, + start_pos, start_pos+length-1, + min_cost, max_cost, 0); + sentence_match[ sentence_id ].push_back( m ); + sentence_match_word_count[ sentence_id ] += length; + + if (max_cost < best_cost) + { + best_cost = max_cost; + if (best_cost == 0) break; + } + //if (match_count >= MAX_MATCH_COUNT) break; + } + } + // cerr << endl; + if (best_cost == 0) break; + //if (match_count >= MAX_MATCH_COUNT) break; + } + // cerr << count << " matches at length " << length << " in " << sentence_match.size() << " tm." << endl; + + if (best_cost == 0) break; + //if (match_count >= MAX_MATCH_COUNT) break; + } + cerr << match_count << " matches in " << sentence_match.size() << " sentences." << endl; + + clock_t clock_matches = clock(); + + // consider each sentence for which we have matches + int old_best_cost = best_cost; + int tm_count_word_match = 0; + int tm_count_word_match2 = 0; + int pruned_match_count = 0; + if (short_match_max_length( input_length )) + { + init_short_matches( input[sentenceInd] ); + } + vector< int > best_tm; + typedef map< int, vector< Match > >::iterator I; + + clock_t clock_validation_sum = 0; + + for(I tm=sentence_match.begin(); tm!=sentence_match.end(); tm++) + { + int tmID = tm->first; + int tm_length = suffixArray.GetSentenceLength(tmID); + vector< Match > &match = tm->second; + add_short_matches( match, source[tmID], input_length, best_cost ); + + //cerr << "match in sentence " << tmID << ": " << match.size() << " [" << tm_length << "]" << endl; + + // quick look: how many words are matched + int words_matched = 0; + for(int m=0;m best_cost) + { + if (length_filter_flag) continue; + } + tm_count_word_match++; + + // prune, check again how many words are matched + vector< Match > pruned = prune_matches( match, best_cost ); + words_matched = 0; + for(int p=0;p best_cost) + { + if (length_filter_flag) continue; + } + tm_count_word_match2++; + + pruned_match_count += pruned.size(); + int prior_best_cost = best_cost; + int cost; + + clock_t clock_validation_start = clock(); + if (! parse_flag || + pruned.size()>=10) // to prevent worst cases + { + string path; + cost = sed( input[sentenceInd], source[tmID], path, false ); + if (cost < best_cost) + { + best_cost = cost; + } + } + + else + { + cost = parse_matches( pruned, input_length, tm_length, best_cost ); + if (prior_best_cost != best_cost) + { + best_tm.clear(); + } + } + clock_validation_sum += clock() - clock_validation_start; + if (cost == best_cost) + { + best_tm.push_back( tmID ); + } + } + cerr << "reduced best cost from " << old_best_cost << " to " << best_cost << endl; + cerr << "tm considered: " << sentence_match.size() + << " word-matched: " << tm_count_word_match + << " word-matched2: " << tm_count_word_match2 + << " best: " << best_tm.size() << endl; + + cerr << "pruned matches: " << ((float)pruned_match_count/(float)tm_count_word_match2) << endl; + + // create xml and extract files + string inputStr, sourceStr; + for (size_t pos = 0; pos < input_length; ++pos) { + inputStr += vocabulary.GetWord(input[sentenceInd][pos]) + " "; + } + + // do not try to find the best ... report multiple matches + if (multiple_flag) { + int input_letter_length = compute_length( input[sentenceInd] ); + for(int si=0; si &sourceSentence = source[s]; + vector &targets = targetAndAlignment[s]; + create_extract(sentenceInd, best_cost, sourceSentence, targets, inputStr, path); + + } + } // if (multiple_flag) + else { + + // find the best matches according to letter sed + string best_path = ""; + int best_match = -1; + int best_letter_cost; + if (lsed_flag) { + best_letter_cost = compute_length( input[sentenceInd] ) * min_match / 100 + 1; + for(int si=0; si 0) { + string path; + sed( input[sentenceInd], source[best_tm[0]], path, false ); + best_path = path; + best_match = best_tm[0]; + } + } + cerr << "elapsed: " << (1000 * (clock()-start_clock) / CLOCKS_PER_SEC) + << " ( range: " << (1000 * (clock_range-start_clock) / CLOCKS_PER_SEC) + << " match: " << (1000 * (clock_matches-clock_range) / CLOCKS_PER_SEC) + << " tm: " << (1000 * (clock()-clock_matches) / CLOCKS_PER_SEC) + << " (validation: " << (1000 * (clock_validation_sum) / CLOCKS_PER_SEC) << ")" + << " )" << endl; + if (lsed_flag) { + cout << best_letter_cost << "/" << compute_length( input[sentenceInd] ) << " ("; + } + cout << best_cost <<"/" << input_length; + if (lsed_flag) cout << ")"; + cout << " ||| " << best_match << " ||| " << best_path << endl; + + // creat xml & extracts + vector &sourceSentence = source[best_match]; + vector &targets = targetAndAlignment[best_match]; + create_extract(sentenceInd, best_cost, sourceSentence, targets, inputStr, best_path); + + } // else if (multiple_flag) + + + } + cerr << "total: " << (1000 * (clock()-start_main_clock) / CLOCKS_PER_SEC) << endl; + +} + +void create_extract(int sentenceInd, int cost, const vector< WORD_ID > &sourceSentence, const vector &targets, const string &inputStr, const string &path) +{ + string sourceStr; + for (size_t pos = 0; pos < sourceSentence.size(); ++pos) { + WORD_ID wordId = sourceSentence[pos]; + sourceStr += vocabulary.GetWord(wordId) + " "; + } + + char *inputFileName = tmpnam(NULL); + ofstream inputFile(inputFileName); + + for (size_t targetInd = 0; targetInd < targets.size(); ++targetInd) { + const SentenceAlignment &sentenceAlignment = targets[targetInd]; + string targetStr = sentenceAlignment.getTargetString(); + string alignStr = sentenceAlignment.getAlignmentString(); + + inputFile + << sentenceInd << endl + << cost << endl + << sourceStr << endl + << inputStr << endl + << targetStr << endl + << alignStr << endl + << path << endl + << sentenceAlignment.count << endl; + + } + + string cmd = string("perl create_xml.perl < ") + inputFileName; + cerr << cmd << endl; + inputFile.close(); + +} diff --git a/contrib/fuzzy-match/fuzzy-match2.h b/contrib/fuzzy-match/fuzzy-match2.h new file mode 100644 index 000000000..614bf971f --- /dev/null +++ b/contrib/fuzzy-match/fuzzy-match2.h @@ -0,0 +1,561 @@ +// +// fuzzy-match2.h +// fuzzy-match +// +// Created by Hieu Hoang on 25/07/2012. +// Copyright 2012 __MyCompanyName__. All rights reserved. +// + +#ifndef fuzzy_match_fuzzy_match2_h +#define fuzzy_match_fuzzy_match2_h + +#include +#include +#include +#include "Vocabulary.h" +#include "SuffixArray.h" +#include "Util.h" +#include "Match.h" + +#define MAX_MATCH_COUNT 10000000 + +Vocabulary vocabulary; + +int basic_flag = false; +int lsed_flag = true; +int refined_flag = true; +int length_filter_flag = true; +int parse_flag = true; +int min_match = 70; +int multiple_flag = false; +int multiple_slack = 0; +int multiple_max = 100; +map< WORD_ID,vector< int > > single_word_index; +// global cache for word pairs +map< pair< WORD_ID, WORD_ID >, unsigned int > lsed; + +void create_extract(int sentenceInd, int cost, const vector< WORD_ID > &sourceSentence, const vector &targets, const string &inputStr, const string &path); + + + +/* Letter string edit distance, e.g. sub 'their' to 'there' costs 2 */ + +unsigned int letter_sed( WORD_ID aIdx, WORD_ID bIdx ) +{ + // check if already computed -> lookup in cache + pair< WORD_ID, WORD_ID > pIdx = make_pair( aIdx, bIdx ); + map< pair< WORD_ID, WORD_ID >, unsigned int >::const_iterator lookup = lsed.find( pIdx ); + if (lookup != lsed.end()) + { + return (lookup->second); + } + + // get surface strings for word indices + const string &a = vocabulary.GetWord( aIdx ); + const string &b = vocabulary.GetWord( bIdx ); + + // initialize cost matrix + unsigned int **cost = (unsigned int**) calloc( sizeof( unsigned int* ), a.size()+1 ); + for( unsigned int i=0; i<=a.size(); i++ ) { + cost[i] = (unsigned int*) calloc( sizeof(unsigned int), b.size()+1 ); + cost[i][0] = i; + } + for( unsigned int j=0; j<=b.size(); j++ ) { + cost[0][j] = j; + } + + // core string edit distance loop + for( unsigned int i=1; i<=a.size(); i++ ) { + for( unsigned int j=1; j<=b.size(); j++ ) { + + unsigned int ins = cost[i-1][j] + 1; + unsigned int del = cost[i][j-1] + 1; + bool match = (a.substr(i-1,1).compare( b.substr(j-1,1) ) == 0); + unsigned int diag = cost[i-1][j-1] + (match ? 0 : 1); + + unsigned int min = (ins < del) ? ins : del; + min = (diag < min) ? diag : min; + + cost[i][j] = min; + } + } + + // clear out memory + unsigned int final = cost[a.size()][b.size()]; + for( unsigned int i=0; i<=a.size(); i++ ) { + free( cost[i] ); + } + free( cost ); + + // cache and return result + lsed[ pIdx ] = final; + return final; +} + +/* string edit distance implementation */ + +unsigned int sed( const vector< WORD_ID > &a, const vector< WORD_ID > &b, string &best_path, bool use_letter_sed ) { + + // initialize cost and path matrices + unsigned int **cost = (unsigned int**) calloc( sizeof( unsigned int* ), a.size()+1 ); + char **path = (char**) calloc( sizeof( char* ), a.size()+1 ); + + for( unsigned int i=0; i<=a.size(); i++ ) { + cost[i] = (unsigned int*) calloc( sizeof(unsigned int), b.size()+1 ); + path[i] = (char*) calloc( sizeof(char), b.size()+1 ); + if (i>0) + { + cost[i][0] = cost[i-1][0]; + if (use_letter_sed) + { + cost[i][0] += vocabulary.GetWord( a[i-1] ).size(); + } + else + { + cost[i][0]++; + } + } + else + { + cost[i][0] = 0; + } + path[i][0] = 'I'; + } + + for( unsigned int j=0; j<=b.size(); j++ ) { + if (j>0) + { + cost[0][j] = cost[0][j-1]; + if (use_letter_sed) + { + cost[0][j] += vocabulary.GetWord( b[j-1] ).size(); + } + else + { + cost[0][j]++; + } + } + else + { + cost[0][j] = 0; + } + path[0][j] = 'D'; + } + + // core string edit distance algorithm + for( unsigned int i=1; i<=a.size(); i++ ) { + for( unsigned int j=1; j<=b.size(); j++ ) { + unsigned int ins = cost[i-1][j]; + unsigned int del = cost[i][j-1]; + unsigned int match; + if (use_letter_sed) + { + ins += vocabulary.GetWord( a[i-1] ).size(); + del += vocabulary.GetWord( b[j-1] ).size(); + match = letter_sed( a[i-1], b[j-1] ); + } + else + { + ins++; + del++; + match = ( a[i-1] == b[j-1] ) ? 0 : 1; + } + unsigned int diag = cost[i-1][j-1] + match; + + char action = (ins < del) ? 'I' : 'D'; + unsigned int min = (ins < del) ? ins : del; + if (diag < min) + { + action = (match>0) ? 'S' : 'M'; + min = diag; + } + + cost[i][j] = min; + path[i][j] = action; + } + } + + // construct string for best path + unsigned int i = a.size(); + unsigned int j = b.size(); + best_path = ""; + while( i>0 || j>0 ) + { + best_path = path[i][j] + best_path; + if (path[i][j] == 'I') + { + i--; + } + else if (path[i][j] == 'D') + { + j--; + } + else + { + i--; + j--; + } + } + + + // clear out memory + unsigned int final = cost[a.size()][b.size()]; + + for( unsigned int i=0; i<=a.size(); i++ ) { + free( cost[i] ); + free( path[i] ); + } + free( cost ); + free( path ); + + // return result + return final; +} + +/* utlility function: compute length of sentence in characters + (spaces do not count) */ + +unsigned int compute_length( const vector< WORD_ID > &sentence ) +{ + unsigned int length = 0; for( unsigned int i=0; i > source, + vector< vector< WORD_ID > > input ) +{ + // go through input set... + for(unsigned int i=0;i= best_cost)) + { + continue; + } + + // compute string edit distance + string path; + unsigned int cost = sed( input[i], source[s], path, use_letter_sed ); + + // update if new best + if (cost < best_cost) + { + best_cost = cost; + best_path = path; + best_match = s; + } + } + cout << best_cost << " ||| " << best_match << " ||| " << best_path << endl; + } +} + +/* definition of short matches + very short n-gram matches (1-grams) will not be looked up in + the suffix array, since there are too many matches + and for longer sentences, at least one 2-gram match must occur */ + +inline int short_match_max_length( int input_length ) +{ + if ( ! refined_flag ) + return 0; + if ( input_length >= 5 ) + return 1; + return 0; +} + +/* if we have non-short matches in a sentence, we need to + take a closer look at it. + this function creates a hash map for all input words and their positions + (to be used by the next function) + (done here, because this has be done only once for an input sentence) */ + +void init_short_matches( const vector< WORD_ID > &input ) +{ + int max_length = short_match_max_length( input.size() ); + if (max_length == 0) + return; + + single_word_index.clear(); + + // store input words and their positions in hash map + for(int i=0; i position_vector; + single_word_index[ input[i] ] = position_vector; + } + single_word_index[ input[i] ].push_back( i ); + } +} + +/* add all short matches to list of matches for a sentence */ + +void add_short_matches( vector< Match > &match, const vector< WORD_ID > &tm, int input_length, int best_cost ) +{ + int max_length = short_match_max_length( input_length ); + if (max_length == 0) + return; + + int tm_length = tm.size(); + map< WORD_ID,vector< int > >::iterator input_word_hit; + for(int t_pos=0; t_pos &position_vector = input_word_hit->second; + for(int j=0; j0 && i_pos == t_pos ) + min_cost++; + + // after match + max_cost += max( (input_length-i_pos) , (tm_length-t_pos)); + min_cost += abs( (input_length-i_pos) - (tm_length-t_pos)); + if ( i_pos != input_length-1 && (input_length-i_pos) == (tm_length-t_pos)) + min_cost++; + + if (min_cost <= best_cost) + { + Match new_match( i_pos,i_pos, t_pos,t_pos, min_cost,max_cost,0 ); + match.push_back( new_match ); + } + } + } + } +} + +/* remove matches that are subsumed by a larger match */ + +vector< Match > prune_matches( const vector< Match > &match, int best_cost ) +{ + //cerr << "\tpruning"; + vector< Match > pruned; + for(int i=match.size()-1; i>=0; i--) + { + //cerr << " (" << match[i].input_start << "," << match[i].input_end + // << " ; " << match[i].tm_start << "," << match[i].tm_end + // << " * " << match[i].min_cost << ")"; + + //if (match[i].min_cost > best_cost) + // continue; + + bool subsumed = false; + for(int j=match.size()-1; j>=0; j--) + { + if (i!=j // do not compare match with itself + && ( match[i].input_end - match[i].input_start <= + match[j].input_end - match[j].input_start ) // i shorter than j + && ((match[i].input_start == match[j].input_start && + match[i].tm_start == match[j].tm_start ) || + (match[i].input_end == match[j].input_end && + match[i].tm_end == match[j].tm_end) ) ) + { + subsumed = true; + } + } + if (! subsumed && match[i].min_cost <= best_cost) + { + //cerr << "*"; + pruned.push_back( match[i] ); + } + } + //cerr << endl; + return pruned; +} + +/* A* parsing method to compute string edit distance */ + +int parse_matches( vector< Match > &match, int input_length, int tm_length, int &best_cost ) +{ + // cerr << "sentence has " << match.size() << " matches, best cost: " << best_cost << ", lengths input: " << input_length << " tm: " << tm_length << endl; + + if (match.size() == 1) + return match[0].max_cost; + if (match.size() == 0) + return input_length+tm_length; + + int this_best_cost = input_length + tm_length; + for(int i=0;i > multi_match; + multi_match.push_back( match ); + + int match_level = 1; + while(multi_match[ match_level-1 ].size()>0) + { + // init vector + vector< Match > empty; + multi_match.push_back( empty ); + + for(int first_level = 0; first_level <= (match_level-1)/2; first_level++) + { + int second_level = match_level - first_level -1; + //cerr << "\tcombining level " << first_level << " and " << second_level << endl; + + vector< Match > &first_match = multi_match[ first_level ]; + vector< Match > &second_match = multi_match[ second_level ]; + + for(int i1 = 0; i1 < first_match.size(); i1++) { + for(int i2 = 0; i2 < second_match.size(); i2++) { + + // do not combine the same pair twice + if (first_level == second_level && i2 <= i1) + { + continue; + } + + // get sorted matches (first is before second) + Match *first, *second; + if (first_match[i1].input_start < second_match[i2].input_start ) + { + first = &first_match[i1]; + second = &second_match[i2]; + } + else + { + second = &first_match[i1]; + first = &second_match[i2]; + } + + //cerr << "\tcombining " + // << "(" << first->input_start << "," << first->input_end << "), " + // << first->tm_start << " [" << first->internal_cost << "]" + // << " with " + // << "(" << second->input_start << "," << second->input_end << "), " + // << second->tm_start<< " [" << second->internal_cost << "]" + // << endl; + + // do not process overlapping matches + if (first->input_end >= second->input_start) + { + continue; + } + + // no overlap / mismatch in tm + if (first->tm_end >= second->tm_start) + { + continue; + } + + // compute cost + int min_cost = 0; + int max_cost = 0; + + // initial + min_cost += abs( first->input_start - first->tm_start ); + max_cost += max( first->input_start, first->tm_start ); + + // same number of words, but not sent. start -> cost is at least 1 + if (first->input_start == first->tm_start && first->input_start > 0) + { + min_cost++; + } + + // in-between + int skipped_words = second->input_start - first->input_end -1; + int skipped_words_tm = second->tm_start - first->tm_end -1; + int internal_cost = max( skipped_words, skipped_words_tm ); + internal_cost += first->internal_cost + second->internal_cost; + min_cost += internal_cost; + max_cost += internal_cost; + + // final + min_cost += abs( (tm_length-1 - second->tm_end) - + (input_length-1 - second->input_end) ); + max_cost += max( (tm_length-1 - second->tm_end), + (input_length-1 - second->input_end) ); + + // same number of words, but not sent. end -> cost is at least 1 + if ( ( input_length-1 - second->input_end + == tm_length-1 - second->tm_end ) + && input_length-1 != second->input_end ) + { + min_cost++; + } + + // cerr << "\tcost: " << min_cost << "-" << max_cost << endl; + + // if worst than best cost, forget it + if (min_cost > best_cost) + { + continue; + } + + // add match + Match new_match( first->input_start, + second->input_end, + first->tm_start, + second->tm_end, + min_cost, + max_cost, + internal_cost); + multi_match[ match_level ].push_back( new_match ); + // cerr << "\tstored\n"; + + // possibly updating this_best_cost + if (max_cost < this_best_cost) + { + // cerr << "\tupdating this best cost to " << max_cost << "\n"; + this_best_cost = max_cost; + + // possibly updating best_cost + if (max_cost < best_cost) + { + // cerr << "\tupdating best cost to " << max_cost << "\n"; + best_cost = max_cost; + } + } + } + } + } + match_level++; + } + return this_best_cost; +} + +#endif diff --git a/contrib/fuzzy-match/make-xml-from-match.perl b/contrib/fuzzy-match/make-xml-from-match.perl new file mode 100644 index 000000000..b5c213a3d --- /dev/null +++ b/contrib/fuzzy-match/make-xml-from-match.perl @@ -0,0 +1,214 @@ +#!/usr/bin/perl -w + +use strict; + +my $DEBUG = 1; + +my $match_file = "tm/BEST.acquis-xml-escaped.4.uniq"; +my $source_file = "data/acquis.truecased.4.en.uniq"; +my $target_file = "data/acquis.truecased.4.fr.uniq.most-frequent"; +my $alignment_file = "data/acquis.truecased.4.align.uniq.most-frequent"; +my $out_file = "data/ac-test.input.xml.4.uniq"; +my $in_file = "evaluation/ac-test.input.tc.4"; + +#my $match_file = "tm/BEST.acquis-xml-escaped.4"; +#my $source_file = "corpus/acquis.truecased.4.en"; +#my $target_file = "corpus/acquis.truecased.4.fr"; +#my $alignment_file = "model/aligned.4.grow-diag-final-and"; +#my $out_file = "data/ac-test.input.xml.4"; +#my $in_file = "evaluation/ac-test.input.tc.4"; + +#my $match_file = "tm/BEST.acquis.with"; +#my $source_file = "../acquis-truecase/corpus/acquis.truecased.190.en"; +#my $target_file = "../acquis-truecase/corpus/acquis.truecased.190.fr"; +#my $alignment_file = "../acquis-truecase/model/aligned.190.grow-diag-final-and"; +#my $out_file = "data/ac-test.input.xml"; +#my $in_file = "evaluation/ac-test.input.tc.1"; + +my @INPUT = `cat $in_file`; chop(@INPUT); +my @SOURCE = `cat $source_file`; chop(@SOURCE); +my @TARGET = `cat $target_file`; chop(@TARGET); +my @ALIGNMENT = `cat $alignment_file`; chop(@ALIGNMENT); + +open(MATCH,$match_file); +open(FRAME,">$out_file"); +for(my $i=0;$i<4107;$i++) { + + # get match data + my $match = ; + chop($match); + my ($score,$sentence,$path) = split(/ \|\|\| /,$match); + + # construct frame + if ($sentence < 1e9 && $sentence >= 0) { + my $frame = &create_xml($SOURCE[$sentence], + $INPUT[$i], + $TARGET[$sentence], + $ALIGNMENT[$sentence], + $path); + print FRAME $frame."\n"; + } + + # no frame -> output source + else { + print FRAME $INPUT[$i]."\n"; + } +} +close(FRAME); +close(MATCH); + +sub create_xml { + my ($source,$input,$target,$alignment,$path) = @_; + + my @INPUT = split(/ /,$input); + my @SOURCE = split(/ /,$source); + my @TARGET = split(/ /,$target); + my %ALIGN = &create_alignment($alignment); + + my %FRAME_INPUT; + my @TARGET_BITMAP; + foreach (@TARGET) { push @TARGET_BITMAP,1 } + + ### STEP 1: FIND MISMATCHES + + my ($s,$i) = (0,0); + my $currently_matching = 0; + my ($start_s,$start_i) = (0,0); + + $path .= "X"; # indicate end + print "$input\n$source\n$target\n$path\n"; + for(my $p=0;$p $#INPUT) { + $start_t = $#TARGET; + } + + # backtrack to previous words if unaligned + if ($start_t == 1000) { + $start_t = -1; + for(my $ss = $s-1; $start_t==-1 && $ss>=0; $ss--) { + foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) { + $start_t = $tt if $tt > $start_t; + } + } + } + $FRAME_INPUT{$start_t} .= $insertion; + } + + $currently_matching = 1; + } + + print "$action $s $i ($start_s $start_i) $currently_matching"; + if ($action ne "I") { + print " ->"; + foreach my $tt (keys %{${$ALIGN{'s'}}[$s]}) { + print " ".$tt; + } + } + print "\n"; + $s++ unless $action eq "I"; + $i++ unless $action eq "D"; + } + + + print $target."\n"; + foreach (@TARGET_BITMAP) { print $_; } print "\n"; + foreach (sort keys %FRAME_INPUT) { + print "$_: $FRAME_INPUT{$_}\n"; + } + + ### STEP 2: BUILD FRAME + + # modify frame + my $frame = ""; + $frame = $FRAME_INPUT{-1} if defined $FRAME_INPUT{-1}; + + my $currently_included = 0; + my $start_t = -1; + push @TARGET_BITMAP,0; # indicate end + + for(my $t=0;$t<=scalar(@TARGET);$t++) { + + # beginning of tm target inclusion + if (!$currently_included && $TARGET_BITMAP[$t]) { + $start_t = $t; + $currently_included = 1; + } + + # end of tm target inclusion (not included word or inserted input) + elsif ($currently_included && + (!$TARGET_BITMAP[$t] || defined($FRAME_INPUT{$t}))) { + # add xml (unless change is at the beginning of the sentence + if ($start_t >= 0) { + my $target = ""; + print "for(tt=$start_t;tt<$t+$TARGET_BITMAP[$t]);\n"; + for(my $tt=$start_t;$tt<$t+$TARGET_BITMAP[$t];$tt++) { + $target .= $TARGET[$tt] . " "; + } + chop($target); + $frame .= " x "; + } + $currently_included = 0; + } + + $frame .= $FRAME_INPUT{$t} if defined $FRAME_INPUT{$t}; + print "$TARGET_BITMAP[$t] $t ($start_t) $currently_included\n"; + } + + print $frame."\n-------------------------------------\n"; + return $frame; +} + +sub create_alignment { + my ($line) = @_; + my (@ALIGNED_TO_S,@ALIGNED_TO_T); + foreach my $point (split(/ /,$line)) { + my ($s,$t) = split(/\-/,$point); + $ALIGNED_TO_S[$s]{$t}++; + $ALIGNED_TO_T[$t]{$s}++; + } + my %ALIGNMENT = ( 's' => \@ALIGNED_TO_S, 't' => \@ALIGNED_TO_T ); + return %ALIGNMENT; +} diff --git a/contrib/fuzzy-match/old/fuzzy-match.cpp b/contrib/fuzzy-match/old/fuzzy-match.cpp new file mode 100644 index 000000000..76c69e246 --- /dev/null +++ b/contrib/fuzzy-match/old/fuzzy-match.cpp @@ -0,0 +1,982 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Vocabulary.h" +#include "SuffixArray.h" + +/** This implementation is explained in + Koehn and Senellart: "Fast Approximate String Matching + with Suffix Arrays and A* Parsing" (AMTA 2010) ***/ + +using namespace std; + +Vocabulary vocabulary; + +int basic_flag = false; +int lsed_flag = true; +int refined_flag = true; +int length_filter_flag = true; +int parse_flag = true; +int min_match = 70; +int multiple_flag = false; +int multiple_slack = 0; +int multiple_max = 100; + +void load_corpus( char* fileName, vector< vector< WORD_ID > > &corpus ) +{ + ifstream fileStream; + fileStream.open(fileName); + if (!fileStream) { + cerr << "file not found: " << fileName << endl; + exit(1); + } + istream *fileStreamP = &fileStream; + + char line[LINE_MAX_LENGTH]; + while(true) + { + SAFE_GETLINE((*fileStreamP), line, LINE_MAX_LENGTH, '\n'); + if (fileStreamP->eof()) break; + corpus.push_back( vocabulary.Tokenize( line ) ); + } +} + + +/* Letter string edit distance, e.g. sub 'their' to 'there' costs 2 */ + +// global cache for word pairs +map< pair< WORD_ID, WORD_ID >, unsigned int > lsed; + +unsigned int letter_sed( WORD_ID aIdx, WORD_ID bIdx ) +{ + // check if already computed -> lookup in cache + pair< WORD_ID, WORD_ID > pIdx = make_pair( aIdx, bIdx ); + map< pair< WORD_ID, WORD_ID >, unsigned int >::const_iterator lookup = lsed.find( pIdx ); + if (lookup != lsed.end()) + { + return (lookup->second); + } + + // get surface strings for word indices + const string &a = vocabulary.GetWord( aIdx ); + const string &b = vocabulary.GetWord( bIdx ); + + // initialize cost matrix + unsigned int **cost = (unsigned int**) calloc( sizeof( unsigned int* ), a.size()+1 ); + for( unsigned int i=0; i<=a.size(); i++ ) { + cost[i] = (unsigned int*) calloc( sizeof(unsigned int), b.size()+1 ); + cost[i][0] = i; + } + for( unsigned int j=0; j<=b.size(); j++ ) { + cost[0][j] = j; + } + + // core string edit distance loop + for( unsigned int i=1; i<=a.size(); i++ ) { + for( unsigned int j=1; j<=b.size(); j++ ) { + + unsigned int ins = cost[i-1][j] + 1; + unsigned int del = cost[i][j-1] + 1; + bool match = (a.substr(i-1,1).compare( b.substr(j-1,1) ) == 0); + unsigned int diag = cost[i-1][j-1] + (match ? 0 : 1); + + unsigned int min = (ins < del) ? ins : del; + min = (diag < min) ? diag : min; + + cost[i][j] = min; + } + } + + // clear out memory + unsigned int final = cost[a.size()][b.size()]; + for( unsigned int i=0; i<=a.size(); i++ ) { + free( cost[i] ); + } + free( cost ); + + // cache and return result + lsed[ pIdx ] = final; + return final; +} + +/* string edit distance implementation */ + +unsigned int sed( const vector< WORD_ID > &a, const vector< WORD_ID > &b, string &best_path, bool use_letter_sed ) { + + // initialize cost and path matrices + unsigned int **cost = (unsigned int**) calloc( sizeof( unsigned int* ), a.size()+1 ); + char **path = (char**) calloc( sizeof( char* ), a.size()+1 ); + + for( unsigned int i=0; i<=a.size(); i++ ) { + cost[i] = (unsigned int*) calloc( sizeof(unsigned int), b.size()+1 ); + path[i] = (char*) calloc( sizeof(char), b.size()+1 ); + if (i>0) + { + cost[i][0] = cost[i-1][0]; + if (use_letter_sed) + { + cost[i][0] += vocabulary.GetWord( a[i-1] ).size(); + } + else + { + cost[i][0]++; + } + } + else + { + cost[i][0] = 0; + } + path[i][0] = 'I'; + } + + for( unsigned int j=0; j<=b.size(); j++ ) { + if (j>0) + { + cost[0][j] = cost[0][j-1]; + if (use_letter_sed) + { + cost[0][j] += vocabulary.GetWord( b[j-1] ).size(); + } + else + { + cost[0][j]++; + } + } + else + { + cost[0][j] = 0; + } + path[0][j] = 'D'; + } + + // core string edit distance algorithm + for( unsigned int i=1; i<=a.size(); i++ ) { + for( unsigned int j=1; j<=b.size(); j++ ) { + unsigned int ins = cost[i-1][j]; + unsigned int del = cost[i][j-1]; + unsigned int match; + if (use_letter_sed) + { + ins += vocabulary.GetWord( a[i-1] ).size(); + del += vocabulary.GetWord( b[j-1] ).size(); + match = letter_sed( a[i-1], b[j-1] ); + } + else + { + ins++; + del++; + match = ( a[i-1] == b[j-1] ) ? 0 : 1; + } + unsigned int diag = cost[i-1][j-1] + match; + + char action = (ins < del) ? 'I' : 'D'; + unsigned int min = (ins < del) ? ins : del; + if (diag < min) + { + action = (match>0) ? 'S' : 'M'; + min = diag; + } + + cost[i][j] = min; + path[i][j] = action; + } + } + + // construct string for best path + unsigned int i = a.size(); + unsigned int j = b.size(); + best_path = ""; + while( i>0 || j>0 ) + { + best_path = path[i][j] + best_path; + if (path[i][j] == 'I') + { + i--; + } + else if (path[i][j] == 'D') + { + j--; + } + else + { + i--; + j--; + } + } + + + // clear out memory + unsigned int final = cost[a.size()][b.size()]; + + for( unsigned int i=0; i<=a.size(); i++ ) { + free( cost[i] ); + free( path[i] ); + } + free( cost ); + free( path ); + + // return result + return final; +} + +/* utlility function: compute length of sentence in characters + (spaces do not count) */ + +unsigned int compute_length( const vector< WORD_ID > &sentence ) +{ + unsigned int length = 0; for( unsigned int i=0; i > source, + vector< vector< WORD_ID > > input ) +{ + // go through input set... + for(unsigned int i=0;i= best_cost)) + { + continue; + } + + // compute string edit distance + string path; + unsigned int cost = sed( input[i], source[s], path, use_letter_sed ); + + // update if new best + if (cost < best_cost) + { + best_cost = cost; + best_path = path; + best_match = s; + } + } + cout << best_cost << " ||| " << best_match << " ||| " << best_path << endl; + } +} + +#define MAX_MATCH_COUNT 10000000 + +/* data structure for n-gram match between input and corpus */ + +class Match { +public: + int input_start; + int input_end; + int tm_start; + int tm_end; + int min_cost; + int max_cost; + int internal_cost; + Match( int is, int ie, int ts, int te, int min, int max, int i ) + :input_start(is), input_end(ie), tm_start(ts), tm_end(te), min_cost(min), max_cost(max), internal_cost(i) + {} +}; + +map< WORD_ID,vector< int > > single_word_index; + +/* definition of short matches + very short n-gram matches (1-grams) will not be looked up in + the suffix array, since there are too many matches + and for longer sentences, at least one 2-gram match must occur */ + +inline int short_match_max_length( int input_length ) +{ + if ( ! refined_flag ) + return 0; + if ( input_length >= 5 ) + return 1; + return 0; +} + +/* if we have non-short matches in a sentence, we need to + take a closer look at it. + this function creates a hash map for all input words and their positions + (to be used by the next function) + (done here, because this has be done only once for an input sentence) */ + +void init_short_matches( const vector< WORD_ID > &input ) +{ + int max_length = short_match_max_length( input.size() ); + if (max_length == 0) + return; + + single_word_index.clear(); + + // store input words and their positions in hash map + for(int i=0; i position_vector; + single_word_index[ input[i] ] = position_vector; + } + single_word_index[ input[i] ].push_back( i ); + } +} + +/* add all short matches to list of matches for a sentence */ + +void add_short_matches( vector< Match > &match, const vector< WORD_ID > &tm, int input_length, int best_cost ) +{ + int max_length = short_match_max_length( input_length ); + if (max_length == 0) + return; + + int tm_length = tm.size(); + map< WORD_ID,vector< int > >::iterator input_word_hit; + for(int t_pos=0; t_pos &position_vector = input_word_hit->second; + for(int j=0; j0 && i_pos == t_pos ) + min_cost++; + + // after match + max_cost += max( (input_length-i_pos) , (tm_length-t_pos)); + min_cost += abs( (input_length-i_pos) - (tm_length-t_pos)); + if ( i_pos != input_length-1 && (input_length-i_pos) == (tm_length-t_pos)) + min_cost++; + + if (min_cost <= best_cost) + { + Match new_match( i_pos,i_pos, t_pos,t_pos, min_cost,max_cost,0 ); + match.push_back( new_match ); + } + } + } + } +} + +/* remove matches that are subsumed by a larger match */ + +vector< Match > prune_matches( const vector< Match > &match, int best_cost ) +{ + //cerr << "\tpruning"; + vector< Match > pruned; + for(int i=match.size()-1; i>=0; i--) + { + //cerr << " (" << match[i].input_start << "," << match[i].input_end + // << " ; " << match[i].tm_start << "," << match[i].tm_end + // << " * " << match[i].min_cost << ")"; + + //if (match[i].min_cost > best_cost) + // continue; + + bool subsumed = false; + for(int j=match.size()-1; j>=0; j--) + { + if (i!=j // do not compare match with itself + && ( match[i].input_end - match[i].input_start <= + match[j].input_end - match[j].input_start ) // i shorter than j + && ((match[i].input_start == match[j].input_start && + match[i].tm_start == match[j].tm_start ) || + (match[i].input_end == match[j].input_end && + match[i].tm_end == match[j].tm_end) ) ) + { + subsumed = true; + } + } + if (! subsumed && match[i].min_cost <= best_cost) + { + //cerr << "*"; + pruned.push_back( match[i] ); + } + } + //cerr << endl; + return pruned; +} + +/* A* parsing method to compute string edit distance */ + +int parse_matches( vector< Match > &match, int input_length, int tm_length, int &best_cost ) +{ + // cerr << "sentence has " << match.size() << " matches, best cost: " << best_cost << ", lengths input: " << input_length << " tm: " << tm_length << endl; + + if (match.size() == 1) + return match[0].max_cost; + if (match.size() == 0) + return input_length+tm_length; + + int this_best_cost = input_length + tm_length; + for(int i=0;i > multi_match; + multi_match.push_back( match ); + + int match_level = 1; + while(multi_match[ match_level-1 ].size()>0) + { + // init vector + vector< Match > empty; + multi_match.push_back( empty ); + + for(int first_level = 0; first_level <= (match_level-1)/2; first_level++) + { + int second_level = match_level - first_level -1; + //cerr << "\tcombining level " << first_level << " and " << second_level << endl; + + vector< Match > &first_match = multi_match[ first_level ]; + vector< Match > &second_match = multi_match[ second_level ]; + + for(int i1 = 0; i1 < first_match.size(); i1++) { + for(int i2 = 0; i2 < second_match.size(); i2++) { + + // do not combine the same pair twice + if (first_level == second_level && i2 <= i1) + { + continue; + } + + // get sorted matches (first is before second) + Match *first, *second; + if (first_match[i1].input_start < second_match[i2].input_start ) + { + first = &first_match[i1]; + second = &second_match[i2]; + } + else + { + second = &first_match[i1]; + first = &second_match[i2]; + } + + //cerr << "\tcombining " + // << "(" << first->input_start << "," << first->input_end << "), " + // << first->tm_start << " [" << first->internal_cost << "]" + // << " with " + // << "(" << second->input_start << "," << second->input_end << "), " + // << second->tm_start<< " [" << second->internal_cost << "]" + // << endl; + + // do not process overlapping matches + if (first->input_end >= second->input_start) + { + continue; + } + + // no overlap / mismatch in tm + if (first->tm_end >= second->tm_start) + { + continue; + } + + // compute cost + int min_cost = 0; + int max_cost = 0; + + // initial + min_cost += abs( first->input_start - first->tm_start ); + max_cost += max( first->input_start, first->tm_start ); + + // same number of words, but not sent. start -> cost is at least 1 + if (first->input_start == first->tm_start && first->input_start > 0) + { + min_cost++; + } + + // in-between + int skipped_words = second->input_start - first->input_end -1; + int skipped_words_tm = second->tm_start - first->tm_end -1; + int internal_cost = max( skipped_words, skipped_words_tm ); + internal_cost += first->internal_cost + second->internal_cost; + min_cost += internal_cost; + max_cost += internal_cost; + + // final + min_cost += abs( (tm_length-1 - second->tm_end) - + (input_length-1 - second->input_end) ); + max_cost += max( (tm_length-1 - second->tm_end), + (input_length-1 - second->input_end) ); + + // same number of words, but not sent. end -> cost is at least 1 + if ( ( input_length-1 - second->input_end + == tm_length-1 - second->tm_end ) + && input_length-1 != second->input_end ) + { + min_cost++; + } + + // cerr << "\tcost: " << min_cost << "-" << max_cost << endl; + + // if worst than best cost, forget it + if (min_cost > best_cost) + { + continue; + } + + // add match + Match new_match( first->input_start, + second->input_end, + first->tm_start, + second->tm_end, + min_cost, + max_cost, + internal_cost); + multi_match[ match_level ].push_back( new_match ); + // cerr << "\tstored\n"; + + // possibly updating this_best_cost + if (max_cost < this_best_cost) + { + // cerr << "\tupdating this best cost to " << max_cost << "\n"; + this_best_cost = max_cost; + + // possibly updating best_cost + if (max_cost < best_cost) + { + // cerr << "\tupdating best cost to " << max_cost << "\n"; + best_cost = max_cost; + } + } + } + } + } + match_level++; + } + return this_best_cost; +} + +int main(int argc, char* argv[]) +{ + vector< vector< WORD_ID > > source, input; + + while(1) { + static struct option long_options[] = { + {"basic", no_argument, &basic_flag, 1}, + {"word", no_argument, &lsed_flag, 0}, + {"unrefined", no_argument, &refined_flag, 0}, + {"nolengthfilter", no_argument, &length_filter_flag, 0}, + {"noparse", no_argument, &parse_flag, 0}, + {"multiple", no_argument, &multiple_flag, 1}, + {"minmatch", required_argument, 0, 'm'}, + {0, 0, 0, 0} + }; + int option_index = 0; + int c = getopt_long (argc, argv, "m:", long_options, &option_index); + if (c == -1) break; + switch (c) { + case 0: +// if (long_options[option_index].flag != 0) +// break; +// printf ("option %s", long_options[option_index].name); +// if (optarg) +// printf (" with arg %s", optarg); +// printf ("\n"); + break; + case 'm': + min_match = atoi(optarg); + if (min_match < 1 || min_match > 100) { + cerr << "error: --minmatch must have value in range 1..100\n"; + exit(1); + } + cerr << "setting min match to " << min_match << endl; + break; + default: + cerr << "usage: syntax: ./fuzzy-match input corpus [--basic] [--word] [--minmatch 1..100]\n"; + exit(1); + } + } + if (lsed_flag) { cerr << "lsed\n"; } + if (basic_flag) { cerr << "basic\n"; } + if (refined_flag) { cerr << "refined\n"; } + if (length_filter_flag) { cerr << "length filter\n"; } + if (parse_flag) { cerr << "parse\n"; } +// exit(1); + + + if (optind+2 != argc) { + cerr << "syntax: ./fuzzy-match input corpus [--basic] [--word] [--minmatch 1..100]\n"; + exit(1); + } + + cerr << "loading corpus...\n"; + + load_corpus(argv[optind], input); + load_corpus(argv[optind+1], source); + + // ./fuzzy-match input corpus [-basic] + +// load_corpus("../corpus/tm.truecased.4.en", source); +// load_corpus("../corpus/tm.truecased.4.it", target); +// load_corpus("../evaluation/test.input.tc.4", input); + +// load_corpus("../../acquis-truecase/corpus/acquis.truecased.190.en", source); +// load_corpus("../../acquis-truecase/evaluation/ac-test.input.tc.190", input); + +// load_corpus("../corpus/tm.truecased.16.en", source); +// load_corpus("../evaluation/test.input.tc.16", input); + + if (basic_flag) { + cerr << "using basic method\n"; + clock_t start_main_clock2 = clock(); + basic_fuzzy_match( source, input ); + cerr << "total: " << (1000 * (clock()-start_main_clock2) / CLOCKS_PER_SEC) << endl; + exit(1); + } + + cerr << "number of input sentences " << input.size() << endl; + + cerr << "creating suffix array...\n"; +// SuffixArray suffixArray( "../corpus/tm.truecased.4.en" ); +// SuffixArray suffixArray( "../../acquis-truecase/corpus/acquis.truecased.190.en" ); + SuffixArray suffixArray( argv[optind+1] ); + + clock_t start_main_clock = clock(); + + // looping through all input sentences... + cerr << "looping...\n"; + for(unsigned int i=0;i > > match_range; + for(size_t start=0;start substring; + bool stillMatched = true; + vector< pair< SuffixArray::INDEX, SuffixArray::INDEX > > matchedAtThisStart; + //cerr << "start: " << start; + for(int word=start; stillMatched && word > sentence_match; + map< int, int > sentence_match_word_count; + + // go through all matches, longest first + for(int length = input[i].size(); length >= 1; length--) + { + // do not create matches, if these are handled by the short match function + if (length <= short_match_max_length( input_length ) ) + { + continue; + } + + unsigned int count = 0; + for(int start = 0; start <= input[i].size() - length; start++) + { + if (match_range[start].size() >= length) + { + pair< SuffixArray::INDEX, SuffixArray::INDEX > &range = match_range[start][length-1]; + // cerr << " (" << range.first << "," << range.second << ")"; + count += range.second - range.first + 1; + + for(SuffixArray::INDEX i=range.first; i<=range.second; i++) + { + int position = suffixArray.GetPosition( i ); + + // sentence length mismatch + size_t sentence_id = suffixArray.GetSentence( position ); + int sentence_length = suffixArray.GetSentenceLength( sentence_id ); + int diff = abs( (int)sentence_length - (int)input_length ); + // cerr << endl << i << "\tsentence " << sentence_id << ", length " << sentence_length; + //if (length <= 2 && input_length>=5 && + // sentence_match.find( sentence_id ) == sentence_match.end()) + // continue; + + if (diff > best_cost) + continue; + + // compute minimal cost + int start_pos = suffixArray.GetWordInSentence( position ); + int end_pos = start_pos + length-1; + // cerr << endl << "\t" << start_pos << "-" << end_pos << " (" << sentence_length << ") vs. " + // << start << "-" << (start+length-1) << " (" << input_length << ")"; + // different number of prior words -> cost is at least diff + int min_cost = abs( start - start_pos ); + + // same number of words, but not sent. start -> cost is at least 1 + if (start == start_pos && start>0) + min_cost++; + + // different number of remaining words -> cost is at least diff + min_cost += abs( ( sentence_length-1 - end_pos ) - + ( input_length-1 - (start+length-1) ) ); + + // same number of words, but not sent. end -> cost is at least 1 + if ( sentence_length-1 - end_pos == + input_length-1 - (start+length-1) + && end_pos != sentence_length-1 ) + min_cost++; + + // cerr << " -> min_cost " << min_cost; + if (min_cost > best_cost) + continue; + + // valid match + match_count++; + + // compute maximal cost + int max_cost = max( start, start_pos ) + + max( sentence_length-1 - end_pos, + input_length-1 - (start+length-1) ); + // cerr << ", max_cost " << max_cost; + + Match m = Match( start, start+length-1, + start_pos, start_pos+length-1, + min_cost, max_cost, 0); + sentence_match[ sentence_id ].push_back( m ); + sentence_match_word_count[ sentence_id ] += length; + + if (max_cost < best_cost) + { + best_cost = max_cost; + if (best_cost == 0) break; + } + //if (match_count >= MAX_MATCH_COUNT) break; + } + } + // cerr << endl; + if (best_cost == 0) break; + //if (match_count >= MAX_MATCH_COUNT) break; + } + // cerr << count << " matches at length " << length << " in " << sentence_match.size() << " tm." << endl; + + if (best_cost == 0) break; + //if (match_count >= MAX_MATCH_COUNT) break; + } + cerr << match_count << " matches in " << sentence_match.size() << " sentences." << endl; + + clock_t clock_matches = clock(); + + // consider each sentence for which we have matches + int old_best_cost = best_cost; + int tm_count_word_match = 0; + int tm_count_word_match2 = 0; + int pruned_match_count = 0; + if (short_match_max_length( input_length )) + { + init_short_matches( input[i] ); + } + vector< int > best_tm; + typedef map< int, vector< Match > >::iterator I; + + clock_t clock_validation_sum = 0; + + for(I tm=sentence_match.begin(); tm!=sentence_match.end(); tm++) + { + int tmID = tm->first; + int tm_length = suffixArray.GetSentenceLength(tmID); + vector< Match > &match = tm->second; + add_short_matches( match, source[tmID], input_length, best_cost ); + + //cerr << "match in sentence " << tmID << ": " << match.size() << " [" << tm_length << "]" << endl; + + // quick look: how many words are matched + int words_matched = 0; + for(int m=0;m best_cost) + { + if (length_filter_flag) continue; + } + tm_count_word_match++; + + // prune, check again how many words are matched + vector< Match > pruned = prune_matches( match, best_cost ); + words_matched = 0; + for(int p=0;p best_cost) + { + if (length_filter_flag) continue; + } + tm_count_word_match2++; + + pruned_match_count += pruned.size(); + int prior_best_cost = best_cost; + int cost; + + clock_t clock_validation_start = clock(); + if (! parse_flag || + pruned.size()>=10) // to prevent worst cases + { + string path; + cost = sed( input[input_id], source[tmID], path, false ); + if (cost < best_cost) + { + best_cost = cost; + } + } + + else + { + cost = parse_matches( pruned, input_length, tm_length, best_cost ); + if (prior_best_cost != best_cost) + { + best_tm.clear(); + } + } + clock_validation_sum += clock() - clock_validation_start; + if (cost == best_cost) + { + best_tm.push_back( tmID ); + } + } + cerr << "reduced best cost from " << old_best_cost << " to " << best_cost << endl; + cerr << "tm considered: " << sentence_match.size() + << " word-matched: " << tm_count_word_match + << " word-matched2: " << tm_count_word_match2 + << " best: " << best_tm.size() << endl; + + cerr << "pruned matches: " << ((float)pruned_match_count/(float)tm_count_word_match2) << endl; + + // do not try to find the best ... report multiple matches + if (multiple_flag) { + int input_letter_length = compute_length( input[input_id] ); + for(int si=0; si 0) { + string path; + sed( input[input_id], source[best_tm[0]], path, false ); + best_path = path; + best_match = best_tm[0]; + } + } + cerr << "elapsed: " << (1000 * (clock()-start_clock) / CLOCKS_PER_SEC) + << " ( range: " << (1000 * (clock_range-start_clock) / CLOCKS_PER_SEC) + << " match: " << (1000 * (clock_matches-clock_range) / CLOCKS_PER_SEC) + << " tm: " << (1000 * (clock()-clock_matches) / CLOCKS_PER_SEC) + << " (validation: " << (1000 * (clock_validation_sum) / CLOCKS_PER_SEC) << ")" + << " )" << endl; + if (lsed_flag) { + cout << best_letter_cost << "/" << compute_length( input[input_id] ) << " ("; + } + cout << best_cost <<"/" << input_length; + if (lsed_flag) cout << ")"; + cout << " ||| " << best_match << " ||| " << best_path << endl; + } + cerr << "total: " << (1000 * (clock()-start_main_clock) / CLOCKS_PER_SEC) << endl; + + +} diff --git a/contrib/fuzzy-match/old/get-multiple-translations-for-uniq-sources.perl b/contrib/fuzzy-match/old/get-multiple-translations-for-uniq-sources.perl new file mode 100644 index 000000000..49e9ce1ec --- /dev/null +++ b/contrib/fuzzy-match/old/get-multiple-translations-for-uniq-sources.perl @@ -0,0 +1,58 @@ +#!/usr/bin/perl -w + +use strict; + +my $src_in = "corpus/acquis.truecased.4.en"; +my $tgt_in = "corpus/acquis.truecased.4.fr"; +my $align_in = "model/aligned.4.grow-diag-final-and"; + +my $src_out = "data/acquis.truecased.4.en.uniq"; +my $tgt_out = "data/acquis.truecased.4.fr.uniq"; +my $tgt_mf = "data/acquis.truecased.4.fr.uniq.most-frequent"; +my $align_out = "data/acquis.truecased.4.align.uniq"; +my $align_mf = "data/acquis.truecased.4.align.uniq.most-frequent"; + +my (%TRANS,%ALIGN); + +open(SRC,$src_in); +open(TGT,$tgt_in); +open(ALIGN,$align_in); +while(my $src = ) { + my $tgt = ; + my $align = ; + chop($tgt); + chop($align); + $TRANS{$src}{$tgt}++; + $ALIGN{$src}{$tgt} = $align; +} +close(SRC); +close(TGT); + +open(SRC_OUT,">$src_out"); +open(TGT_OUT,">$tgt_out"); +open(TGT_MF, ">$tgt_mf"); +open(ALIGN_OUT,">$align_out"); +open(ALIGN_MF, ">$align_mf"); +foreach my $src (keys %TRANS) { + print SRC_OUT $src; + my $first = 1; + my ($max,$best) = (0); + foreach my $tgt (keys %{$TRANS{$src}}) { + print TGT_OUT " ||| " unless $first; + print TGT_OUT $TRANS{$src}{$tgt}." ".$tgt; + print ALIGN_OUT " ||| " unless $first; + print ALIGN_OUT $ALIGN{$src}{$tgt}; + if ($TRANS{$src}{$tgt} > $max) { + $max = $TRANS{$src}{$tgt}; + $best = $tgt; + } + $first = 0; + } + print TGT_OUT "\n"; + print ALIGN_OUT "\n"; + print TGT_MF $best."\n"; + print ALIGN_MF $ALIGN{$src}{$best}."\n"; +} +close(SRC_OUT); +close(TGT_OUT); + diff --git a/contrib/fuzzy-match/old/make-pt-from-tm.perl b/contrib/fuzzy-match/old/make-pt-from-tm.perl new file mode 100755 index 000000000..6bdb2fa93 --- /dev/null +++ b/contrib/fuzzy-match/old/make-pt-from-tm.perl @@ -0,0 +1,308 @@ +#!/usr/bin/perl -w + +use strict; +use FindBin qw($RealBin); +use File::Basename; + +my $DEBUG = 1; +my $OUTPUT_RULES = 1; + +#my $data_root = "/Users/hieuhoang/workspace/experiment/data/tm-mt-integration/"; +my $in_file = $ARGV[0]; #"$data_root/in/ac-test.input.tc.4"; +my $source_file = $ARGV[1]; #"$data_root/in/acquis.truecased.4.en.uniq"; +my $target_file = $ARGV[2]; #"$data_root/in/acquis.truecased.4.fr.uniq"; +my $alignment_file = $ARGV[3]; #"$data_root/in/acquis.truecased.4.align.uniq"; +my $lex_file = $ARGV[4]; #$data_root/in/lex.4; +my $pt_file = $ARGV[5]; #"$data_root/out/pt"; + +my $cmd; + +my $TMPDIR=dirname($pt_file) ."/tmp.$$"; +$cmd = "mkdir -p $TMPDIR"; +`$cmd`; + +my $match_file = "$TMPDIR/match"; + +# suffix array creation and extraction +$cmd = "$RealBin/fuzzy-match --multiple $in_file $source_file > $match_file"; +print STDERR "$cmd \n"; +`$cmd`; + +# make into xml and pt +my $out_file = "$TMPDIR/ac-test.input.xml.4.uniq.multi.tuning"; + +my @INPUT = `cat $in_file`; chop(@INPUT); +my @ALL_SOURCE = `cat $source_file`; chop(@ALL_SOURCE); +my @ALL_TARGET = `cat $target_file`; chop(@ALL_TARGET); +my @ALL_ALIGNMENT = `cat $alignment_file`; chop(@ALL_ALIGNMENT); + +open(MATCH,$match_file); +open(FRAME,">$out_file"); +open(RULE,">$out_file.extract") if $OUTPUT_RULES; +open(RULE_INV,">$out_file.extract.inv") if $OUTPUT_RULES; +open(INFO,">$out_file.info"); +while( my $match = ) { + chop($match); + my ($score,$sentence,$path) = split(/ \|\|\| /,$match); + + $score =~ /^(\d+) (.+)/ || die; + my ($i,$match_score) = ($1,$2); + print STDERR "i=$i match_score=$match_score\n"; + + # construct frame + if ($sentence < 1e9 && $sentence >= 0) { + my $SOURCE = $ALL_SOURCE[$sentence]; + my @ALIGNMENT = split(/ \|\|\| /,$ALL_ALIGNMENT[$sentence]); + my @TARGET = split(/ \|\|\| /,$ALL_TARGET[$sentence]); + + for(my $j=0;$j $out_file.extract.sorted.gz`; +`LC_ALL=C sort $out_file.extract.inv | gzip -c > $out_file.extract.inv.sorted.gz`; + +if ($OUTPUT_RULES) +{ + $cmd = "$RealBin/../../scripts/training/train-model.perl -dont-zip -first-step 6 -last-step 6 -f en -e fr -hierarchical -extract-file $out_file.extract -lexical-file $lex_file -phrase-translation-table $pt_file"; + print STDERR "Executing: $cmd \n"; + `$cmd`; +} + +#$cmd = "rm -rf $TMPDIR"; +#`$cmd`; + +####################################################### +sub create_xml { + my ($source,$input,$target,$alignment,$path) = @_; + + print STDERR " HIEU \n $source \n $input \n $target \n $alignment \n $path \n"; + + my @INPUT = split(/ /,$input); + my @SOURCE = split(/ /,$source); + my @TARGET = split(/ /,$target); + my %ALIGN = &create_alignment($alignment); + + my %FRAME_INPUT; + my (@NT,@INPUT_BITMAP,@TARGET_BITMAP,%ALIGNMENT_I_TO_S); + foreach (@TARGET) { push @TARGET_BITMAP,1 } + + ### STEP 1: FIND MISMATCHES + + my ($s,$i) = (0,0); + my $currently_matching = 0; + my ($start_s,$start_i) = (0,0); + + $path .= "X"; # indicate end + print STDERR "$input\n$source\n$target\n$path\n"; + for(my $p=0;$p $#INPUT) { + $start_t = $#TARGET; + } + + # backtrack to previous words if unaligned + if ($start_t == 1000) { + $start_t = -1; + for(my $ss = $s-1; $start_t==-1 && $ss>=0; $ss--) { + foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) { + $start_t = $tt if $tt > $start_t; + } + } + } + $FRAME_INPUT{$start_t} .= $insertion; + my %NT = ("start_t" => $start_t, + "start_i" => $start_i ); + push @NT,\%NT; + } + $currently_matching = 1; + } + + print STDERR "$action $s $i ($start_s $start_i) $currently_matching"; + if ($action ne "I") { + print STDERR " ->"; + foreach my $tt (keys %{${$ALIGN{'s'}}[$s]}) { + print STDERR " ".$tt; + } + } + print STDERR "\n"; + $s++ unless $action eq "I"; + $i++ unless $action eq "D"; + $ALIGNMENT_I_TO_S{$i} = $s unless $action eq "D"; + push @INPUT_BITMAP, 1 if $action eq "M"; + push @INPUT_BITMAP, 0 if $action eq "I" || $action eq "S"; + } + + + print STDERR $target."\n"; + foreach (@TARGET_BITMAP) { print STDERR $_; } print STDERR "\n"; + foreach (sort keys %FRAME_INPUT) { + print STDERR "$_: $FRAME_INPUT{$_}\n"; + } + + ### STEP 2: BUILD RULE AND FRAME + + # hierarchical rule + my $rule_s = ""; + my $rule_pos_s = 0; + my %RULE_ALIGNMENT_S; + for(my $i=0;$i=0 && $TARGET_BITMAP[$t]) { + $rule_t .= $TARGET[$t]." "; + $RULE_ALIGNMENT_T{$t} = $rule_pos_t++; + } + foreach my $NT (@NT) { + if ($t == $$NT{"start_t"}) { + $rule_t .= "[X][X] "; + $$NT{"rule_pos_t"} = $rule_pos_t++; + } + } + } + + my $rule_alignment = ""; + foreach my $s (sort { $a <=> $b} keys %RULE_ALIGNMENT_S) { + foreach my $t (keys %{$ALIGN{"s"}[$s]}) { + next unless defined($RULE_ALIGNMENT_T{$t}); + $rule_alignment .= $RULE_ALIGNMENT_S{$s}."-".$RULE_ALIGNMENT_T{$t}." "; + } + } + foreach my $NT (@NT) { + $rule_alignment .= $$NT{"rule_pos_s"}."-".$$NT{"rule_pos_t"}." "; + } + + chop($rule_s); + chop($rule_t); + chop($rule_alignment); + + my $rule_alignment_inv = ""; + foreach (split(/ /,$rule_alignment)) { + /^(\d+)\-(\d+)$/; + $rule_alignment_inv .= "$2-$1 "; + } + chop($rule_alignment_inv); + + # frame + my $frame = ""; + $frame = $FRAME_INPUT{-1} if defined $FRAME_INPUT{-1}; + + my $currently_included = 0; + my $start_t = -1; + push @TARGET_BITMAP,0; # indicate end + + for(my $t=0;$t<=scalar(@TARGET);$t++) { + # beginning of tm target inclusion + if (!$currently_included && $TARGET_BITMAP[$t]) { + $start_t = $t; + $currently_included = 1; + } + + # end of tm target inclusion (not included word or inserted input) + elsif ($currently_included && + (!$TARGET_BITMAP[$t] || defined($FRAME_INPUT{$t}))) { + # add xml (unless change is at the beginning of the sentence + if ($start_t >= 0) { + my $target = ""; + print STDERR "for(tt=$start_t;tt<$t+$TARGET_BITMAP[$t]);\n"; + for(my $tt=$start_t;$tt<$t+$TARGET_BITMAP[$t];$tt++) { + $target .= $TARGET[$tt] . " "; + } + chop($target); + $frame .= " x "; + } + $currently_included = 0; + } + + $frame .= $FRAME_INPUT{$t} if defined $FRAME_INPUT{$t}; + print STDERR "$TARGET_BITMAP[$t] $t ($start_t) $currently_included\n"; + } + + print STDERR $frame."\n-------------------------------------\n"; + return ($frame,$rule_s,$rule_t,$rule_alignment,$rule_alignment_inv); +} + +sub create_alignment { + my ($line) = @_; + my (@ALIGNED_TO_S,@ALIGNED_TO_T); + foreach my $point (split(/ /,$line)) { + my ($s,$t) = split(/\-/,$point); + $ALIGNED_TO_S[$s]{$t}++; + $ALIGNED_TO_T[$t]{$s}++; + } + my %ALIGNMENT = ( 's' => \@ALIGNED_TO_S, 't' => \@ALIGNED_TO_T ); + return %ALIGNMENT; +} diff --git a/contrib/fuzzy-match/old/make-pt-from-tm2.perl b/contrib/fuzzy-match/old/make-pt-from-tm2.perl new file mode 100755 index 000000000..3a5fa4171 --- /dev/null +++ b/contrib/fuzzy-match/old/make-pt-from-tm2.perl @@ -0,0 +1,300 @@ +#!/usr/bin/perl -w -d + +use strict; +use FindBin qw($RealBin); +use File::Basename; + +my $DEBUG = 1; +my $OUTPUT_RULES = 1; + +#my $data_root = "/Users/hieuhoang/workspace/experiment/data/tm-mt-integration/"; +my $in_file = $ARGV[0]; #"$data_root/in/ac-test.input.tc.4"; +my $source_file = $ARGV[1]; #"$data_root/in/acquis.truecased.4.en.uniq"; +my $target_file = $ARGV[2]; #"$data_root/in/acquis.truecased.4.fr.uniq"; +my $alignment_file = $ARGV[3]; #"$data_root/in/acquis.truecased.4.align.uniq"; +my $lex_file = $ARGV[4]; #$data_root/in/lex.4; +my $pt_file = $ARGV[5]; #"$data_root/out/pt"; + +my $cmd; + +my $TMPDIR= "/tmp/tmp.$$"; +$cmd = "mkdir -p $TMPDIR"; +`$cmd`; +$TMPDIR = "/Users/hieuhoang/workspace/experiment/data/tm-mt-integration/out/tmp.3196"; + +my $match_file = "$TMPDIR/match"; + +# suffix array creation and extraction +$cmd = "$RealBin/fuzzy-match --multiple $in_file $source_file > $match_file"; +`$cmd`; + +# make into xml and pt +my $out_file = "$TMPDIR/ac-test.input.xml.4.uniq.multi.tuning"; + +open(MATCH,$match_file); +open(FRAME,">$out_file"); +open(RULE,">$out_file.extract") if $OUTPUT_RULES; +open(RULE_INV,">$out_file.extract.inv") if $OUTPUT_RULES; +open(INFO,">$out_file.info"); +while( my $match = ) { + chop($match); + my ($score,$sentence,$path) = split(/ \|\|\| /,$match); + + $score =~ /^(\d+) (.+)/ || die; + my ($i,$match_score) = ($1,$2); + + # construct frame + if ($sentence < 1e9 && $sentence >= 0) { + my $SOURCE = $ALL_SOURCE[$sentence]; + my @ALIGNMENT = split(/ \|\|\| /,$ALL_ALIGNMENT[$sentence]); + my @TARGET = split(/ \|\|\| /,$ALL_TARGET[$sentence]); + + for(my $j=0;$j $out_file.extract.sorted.gz`; +`LC_ALL=C sort $out_file.extract.inv | gzip -c > $out_file.extract.inv.sorted.gz`; + +if ($OUTPUT_RULES) +{ + $cmd = "$RealBin/../../scripts/training/train-model.perl -dont-zip -first-step 6 -last-step 6 -f en -e fr -hierarchical -extract-file $out_file.extract -lexical-file $lex_file -phrase-translation-table $pt_file"; + print STDERR "Executing: $cmd \n"; + `$cmd`; +} + +#$cmd = "rm -rf $TMPDIR"; +#`$cmd`; + +####################################################### +sub create_xml { + my ($source,$input,$target,$alignment,$path) = @_; + + my @INPUT = split(/ /,$input); + my @SOURCE = split(/ /,$source); + my @TARGET = split(/ /,$target); + my %ALIGN = &create_alignment($alignment); + + my %FRAME_INPUT; + my (@NT,@INPUT_BITMAP,@TARGET_BITMAP,%ALIGNMENT_I_TO_S); + foreach (@TARGET) { push @TARGET_BITMAP,1 } + + ### STEP 1: FIND MISMATCHES + + my ($s,$i) = (0,0); + my $currently_matching = 0; + my ($start_s,$start_i) = (0,0); + + $path .= "X"; # indicate end + print STDERR "$input\n$source\n$target\n$path\n"; + for(my $p=0;$p $#INPUT) { + $start_t = $#TARGET; + } + + # backtrack to previous words if unaligned + if ($start_t == 1000) { + $start_t = -1; + for(my $ss = $s-1; $start_t==-1 && $ss>=0; $ss--) { + foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) { + $start_t = $tt if $tt > $start_t; + } + } + } + $FRAME_INPUT{$start_t} .= $insertion; + my %NT = ("start_t" => $start_t, + "start_i" => $start_i ); + push @NT,\%NT; + } + $currently_matching = 1; + } + + print STDERR "$action $s $i ($start_s $start_i) $currently_matching"; + if ($action ne "I") { + print STDERR " ->"; + foreach my $tt (keys %{${$ALIGN{'s'}}[$s]}) { + print STDERR " ".$tt; + } + } + print STDERR "\n"; + $s++ unless $action eq "I"; + $i++ unless $action eq "D"; + $ALIGNMENT_I_TO_S{$i} = $s unless $action eq "D"; + push @INPUT_BITMAP, 1 if $action eq "M"; + push @INPUT_BITMAP, 0 if $action eq "I" || $action eq "S"; + } + + + print STDERR $target."\n"; + foreach (@TARGET_BITMAP) { print STDERR $_; } print STDERR "\n"; + foreach (sort keys %FRAME_INPUT) { + print STDERR "$_: $FRAME_INPUT{$_}\n"; + } + + ### STEP 2: BUILD RULE AND FRAME + + # hierarchical rule + my $rule_s = ""; + my $rule_pos_s = 0; + my %RULE_ALIGNMENT_S; + for(my $i=0;$i=0 && $TARGET_BITMAP[$t]) { + $rule_t .= $TARGET[$t]." "; + $RULE_ALIGNMENT_T{$t} = $rule_pos_t++; + } + foreach my $NT (@NT) { + if ($t == $$NT{"start_t"}) { + $rule_t .= "[X][X] "; + $$NT{"rule_pos_t"} = $rule_pos_t++; + } + } + } + + my $rule_alignment = ""; + foreach my $s (sort { $a <=> $b} keys %RULE_ALIGNMENT_S) { + foreach my $t (keys %{$ALIGN{"s"}[$s]}) { + next unless defined($RULE_ALIGNMENT_T{$t}); + $rule_alignment .= $RULE_ALIGNMENT_S{$s}."-".$RULE_ALIGNMENT_T{$t}." "; + } + } + foreach my $NT (@NT) { + $rule_alignment .= $$NT{"rule_pos_s"}."-".$$NT{"rule_pos_t"}." "; + } + + chop($rule_s); + chop($rule_t); + chop($rule_alignment); + + my $rule_alignment_inv = ""; + foreach (split(/ /,$rule_alignment)) { + /^(\d+)\-(\d+)$/; + $rule_alignment_inv .= "$2-$1 "; + } + chop($rule_alignment_inv); + + # frame + my $frame = ""; + $frame = $FRAME_INPUT{-1} if defined $FRAME_INPUT{-1}; + + my $currently_included = 0; + my $start_t = -1; + push @TARGET_BITMAP,0; # indicate end + + for(my $t=0;$t<=scalar(@TARGET);$t++) { + # beginning of tm target inclusion + if (!$currently_included && $TARGET_BITMAP[$t]) { + $start_t = $t; + $currently_included = 1; + } + + # end of tm target inclusion (not included word or inserted input) + elsif ($currently_included && + (!$TARGET_BITMAP[$t] || defined($FRAME_INPUT{$t}))) { + # add xml (unless change is at the beginning of the sentence + if ($start_t >= 0) { + my $target = ""; + print STDERR "for(tt=$start_t;tt<$t+$TARGET_BITMAP[$t]);\n"; + for(my $tt=$start_t;$tt<$t+$TARGET_BITMAP[$t];$tt++) { + $target .= $TARGET[$tt] . " "; + } + chop($target); + $frame .= " x "; + } + $currently_included = 0; + } + + $frame .= $FRAME_INPUT{$t} if defined $FRAME_INPUT{$t}; + print STDERR "$TARGET_BITMAP[$t] $t ($start_t) $currently_included\n"; + } + + print STDERR $frame."\n-------------------------------------\n"; + return ($frame,$rule_s,$rule_t,$rule_alignment,$rule_alignment_inv); +} + +sub create_alignment { + my ($line) = @_; + my (@ALIGNED_TO_S,@ALIGNED_TO_T); + foreach my $point (split(/ /,$line)) { + my ($s,$t) = split(/\-/,$point); + $ALIGNED_TO_S[$s]{$t}++; + $ALIGNED_TO_T[$t]{$s}++; + } + my %ALIGNMENT = ( 's' => \@ALIGNED_TO_S, 't' => \@ALIGNED_TO_T ); + return %ALIGNMENT; +} diff --git a/contrib/fuzzy-match/old/make-xml-from-match-multiple.perl b/contrib/fuzzy-match/old/make-xml-from-match-multiple.perl new file mode 100755 index 000000000..e16c9de75 --- /dev/null +++ b/contrib/fuzzy-match/old/make-xml-from-match-multiple.perl @@ -0,0 +1,288 @@ +#!/usr/bin/perl -w + +use strict; + +my $DEBUG = 1; +my $OUTPUT_RULES = 1; + +my $scripts_root_dir = "/Users/hieuhoang/workspace/github/hieuhoang/scripts"; + +my $data_root = "/Users/hieuhoang/workspace/experiment/data/tm-mt-integration/"; +#my $match_file = "$data_root/in/BEST.acquis-xml-escaped.4.uniq.multi.tuning"; +my $match_file = "$data_root/out/BEST"; +my $source_file = "$data_root/in/acquis.truecased.4.en.uniq"; +my $target_file = "$data_root/in/acquis.truecased.4.fr.uniq"; +my $alignment_file = "$data_root/in/acquis.truecased.4.align.uniq"; +my $out_file = "$data_root/out/ac-test.input.xml.4.uniq.multi.tuning"; +my $in_file = "$data_root/in/ac-test.input.tc.4"; + +#my $match_file = "tm/BEST.acquis-xml-escaped.4.uniq.multi"; +#my $source_file = "data/acquis.truecased.4.en.uniq"; +#my $target_file = "data/acquis.truecased.4.fr.uniq"; +#my $alignment_file = "data/acquis.truecased.4.align.uniq"; +#my $out_file = "data/ac-test.input.xml.4.uniq.multi.xxx"; +#my $in_file = "evaluation/ac-test.input.tc.4"; + +my @INPUT = `cat $in_file`; chop(@INPUT); +my @ALL_SOURCE = `cat $source_file`; chop(@ALL_SOURCE); +my @ALL_TARGET = `cat $target_file`; chop(@ALL_TARGET); +my @ALL_ALIGNMENT = `cat $alignment_file`; chop(@ALL_ALIGNMENT); + +open(MATCH,$match_file); +open(FRAME,">$out_file"); +open(RULE,">$out_file.extract") if $OUTPUT_RULES; +open(RULE_INV,">$out_file.extract.inv") if $OUTPUT_RULES; +open(INFO,">$out_file.info"); +while( my $match = ) { + chop($match); + my ($score,$sentence,$path) = split(/ \|\|\| /,$match); + + $score =~ /^(\d+) (.+)/ || die; + my ($i,$match_score) = ($1,$2); + + # construct frame + if ($sentence < 1e9 && $sentence >= 0) { + my $SOURCE = $ALL_SOURCE[$sentence]; + my @ALIGNMENT = split(/ \|\|\| /,$ALL_ALIGNMENT[$sentence]); + my @TARGET = split(/ \|\|\| /,$ALL_TARGET[$sentence]); + + for(my $j=0;$j $out_file.extract.sorted.gz`; +`LC_ALL=C sort $out_file.extract.inv | gzip -c > $out_file.extract.inv.sorted.gz`; + +`$scripts_root_dir/training/train-model.perl -dont-zip -first-step 6 -last-step 6 -f en -e fr -hierarchical -extract-file $out_file.extract -lexical-file $data_root/in/lex.4 -phrase-translation-table $out_file.phrase-table` if $OUTPUT_RULES; + +sub create_xml { + my ($source,$input,$target,$alignment,$path) = @_; + + my @INPUT = split(/ /,$input); + my @SOURCE = split(/ /,$source); + my @TARGET = split(/ /,$target); + my %ALIGN = &create_alignment($alignment); + + my %FRAME_INPUT; + my (@NT,@INPUT_BITMAP,@TARGET_BITMAP,%ALIGNMENT_I_TO_S); + foreach (@TARGET) { push @TARGET_BITMAP,1 } + + ### STEP 1: FIND MISMATCHES + + my ($s,$i) = (0,0); + my $currently_matching = 0; + my ($start_s,$start_i) = (0,0); + + $path .= "X"; # indicate end + print "$input\n$source\n$target\n$path\n"; + for(my $p=0;$p $#INPUT) { + $start_t = $#TARGET; + } + + # backtrack to previous words if unaligned + if ($start_t == 1000) { + $start_t = -1; + for(my $ss = $s-1; $start_t==-1 && $ss>=0; $ss--) { + foreach my $tt (keys %{${$ALIGN{'s'}}[$ss]}) { + $start_t = $tt if $tt > $start_t; + } + } + } + $FRAME_INPUT{$start_t} .= $insertion; + my %NT = ("start_t" => $start_t, + "start_i" => $start_i ); + push @NT,\%NT; + } + $currently_matching = 1; + } + + print "$action $s $i ($start_s $start_i) $currently_matching"; + if ($action ne "I") { + print " ->"; + foreach my $tt (keys %{${$ALIGN{'s'}}[$s]}) { + print " ".$tt; + } + } + print "\n"; + $s++ unless $action eq "I"; + $i++ unless $action eq "D"; + $ALIGNMENT_I_TO_S{$i} = $s unless $action eq "D"; + push @INPUT_BITMAP, 1 if $action eq "M"; + push @INPUT_BITMAP, 0 if $action eq "I" || $action eq "S"; + } + + + print $target."\n"; + foreach (@TARGET_BITMAP) { print $_; } print "\n"; + foreach (sort keys %FRAME_INPUT) { + print "$_: $FRAME_INPUT{$_}\n"; + } + + ### STEP 2: BUILD RULE AND FRAME + + # hierarchical rule + my $rule_s = ""; + my $rule_pos_s = 0; + my %RULE_ALIGNMENT_S; + for(my $i=0;$i=0 && $TARGET_BITMAP[$t]) { + $rule_t .= $TARGET[$t]." "; + $RULE_ALIGNMENT_T{$t} = $rule_pos_t++; + } + foreach my $NT (@NT) { + if ($t == $$NT{"start_t"}) { + $rule_t .= "[X][X] "; + $$NT{"rule_pos_t"} = $rule_pos_t++; + } + } + } + + my $rule_alignment = ""; + foreach my $s (sort { $a <=> $b} keys %RULE_ALIGNMENT_S) { + foreach my $t (keys %{$ALIGN{"s"}[$s]}) { + next unless defined($RULE_ALIGNMENT_T{$t}); + $rule_alignment .= $RULE_ALIGNMENT_S{$s}."-".$RULE_ALIGNMENT_T{$t}." "; + } + } + foreach my $NT (@NT) { + $rule_alignment .= $$NT{"rule_pos_s"}."-".$$NT{"rule_pos_t"}." "; + } + + chop($rule_s); + chop($rule_t); + chop($rule_alignment); + + my $rule_alignment_inv = ""; + foreach (split(/ /,$rule_alignment)) { + /^(\d+)\-(\d+)$/; + $rule_alignment_inv .= "$2-$1 "; + } + chop($rule_alignment_inv); + + # frame + my $frame = ""; + $frame = $FRAME_INPUT{-1} if defined $FRAME_INPUT{-1}; + + my $currently_included = 0; + my $start_t = -1; + push @TARGET_BITMAP,0; # indicate end + + for(my $t=0;$t<=scalar(@TARGET);$t++) { + # beginning of tm target inclusion + if (!$currently_included && $TARGET_BITMAP[$t]) { + $start_t = $t; + $currently_included = 1; + } + + # end of tm target inclusion (not included word or inserted input) + elsif ($currently_included && + (!$TARGET_BITMAP[$t] || defined($FRAME_INPUT{$t}))) { + # add xml (unless change is at the beginning of the sentence + if ($start_t >= 0) { + my $target = ""; + print "for(tt=$start_t;tt<$t+$TARGET_BITMAP[$t]);\n"; + for(my $tt=$start_t;$tt<$t+$TARGET_BITMAP[$t];$tt++) { + $target .= $TARGET[$tt] . " "; + } + chop($target); + $frame .= " x "; + } + $currently_included = 0; + } + + $frame .= $FRAME_INPUT{$t} if defined $FRAME_INPUT{$t}; + print "$TARGET_BITMAP[$t] $t ($start_t) $currently_included\n"; + } + + print $frame."\n-------------------------------------\n"; + return ($frame,$rule_s,$rule_t,$rule_alignment,$rule_alignment_inv); +} + +sub create_alignment { + my ($line) = @_; + my (@ALIGNED_TO_S,@ALIGNED_TO_T); + foreach my $point (split(/ /,$line)) { + my ($s,$t) = split(/\-/,$point); + $ALIGNED_TO_S[$s]{$t}++; + $ALIGNED_TO_T[$t]{$s}++; + } + my %ALIGNMENT = ( 's' => \@ALIGNED_TO_S, 't' => \@ALIGNED_TO_T ); + return %ALIGNMENT; +} diff --git a/contrib/fuzzy-match/suffix-test.cpp b/contrib/fuzzy-match/suffix-test.cpp new file mode 100644 index 000000000..01b722fb4 --- /dev/null +++ b/contrib/fuzzy-match/suffix-test.cpp @@ -0,0 +1,27 @@ +#include "SuffixArray.h" + +using namespace std; + +int main(int argc, char* argv[]) +{ + SuffixArray suffixArray( "/home/pkoehn/syntax/grammars/wmt09-de-en/corpus.1k.de" ); + //suffixArray.List(10,20); + vector< string > der; + der.push_back("der"); + vector< string > inDer; + inDer.push_back("in"); + inDer.push_back("der"); + vector< string > zzz; + zzz.push_back("zzz"); + vector< string > derDer; + derDer.push_back("der"); + derDer.push_back("der"); + + cout << "count of 'der' " << suffixArray.Count( der ) << endl; + cout << "limited count of 'der' " << suffixArray.MinCount( der, 2 ) << endl; + cout << "count of 'in der' " << suffixArray.Count( inDer ) << endl; + cout << "count of 'der der' " << suffixArray.Count( derDer ) << endl; + cout << "limited count of 'der der' " << suffixArray.MinCount( derDer, 1 ) << endl; + // cout << "count of 'zzz' " << suffixArray.Count( zzz ) << endl; + // cout << "limited count of 'zzz' " << suffixArray.LimitedCount( zzz, 1 ) << endl; +} diff --git a/contrib/lmserver/INSTALL b/contrib/lmserver/INSTALL deleted file mode 120000 index 81fa6ffa4..000000000 --- a/contrib/lmserver/INSTALL +++ /dev/null @@ -1 +0,0 @@ -/usr/share/automake-1.9/INSTALL \ No newline at end of file diff --git a/contrib/other-builds/CreateOnDisk.vcxproj b/contrib/other-builds/CreateOnDisk.vcxproj index b3f94ed7e..10073b7fe 100644 --- a/contrib/other-builds/CreateOnDisk.vcxproj +++ b/contrib/other-builds/CreateOnDisk.vcxproj @@ -43,6 +43,8 @@ $(SolutionDir)$(Configuration)\ $(Configuration)\ false + C:\Program Files\boost\boost_1_47;$(IncludePath) + C:\Program Files\boost\boost_1_47;$(IncludePath) @@ -58,10 +60,11 @@ C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../..;%(AdditionalIncludeDirectories) - zdll.lib;$(SolutionDir)/$(Configuration)/moses.lib;$(SolutionDir)/$(Configuration)/kenlm.lib;$(SolutionDir)/$(Configuration)/OnDiskPt.lib;%(AdditionalDependencies) + C:\GnuWin32\lib\zlib.lib;$(SolutionDir)/$(Configuration)/moses.lib;$(SolutionDir)/$(Configuration)/kenlm.lib;$(SolutionDir)/$(Configuration)/OnDiskPt.lib;%(AdditionalDependencies) true Console MachineX86 + C:\boost\boost_1_47\lib;%(AdditionalLibraryDirectories) @@ -69,7 +72,7 @@ MaxSpeed true WITH_THREADS;NO_PIPES;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreaded + MultiThreadedDLL true @@ -78,12 +81,13 @@ C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../..;%(AdditionalIncludeDirectories) - zdll.lib;$(SolutionDir)/$(Configuration)/moses.lib;$(SolutionDir)/$(Configuration)/kenlm.lib;$(SolutionDir)/$(Configuration)/OnDiskPt.lib;%(AdditionalDependencies) + C:\GnuWin32\lib\zlib.lib;$(SolutionDir)/$(Configuration)/moses.lib;$(SolutionDir)/$(Configuration)/kenlm.lib;$(SolutionDir)/$(Configuration)/OnDiskPt.lib;%(AdditionalDependencies) true Console true true MachineX86 + C:\boost\boost_1_47\lib;%(AdditionalLibraryDirectories) diff --git a/contrib/other-builds/OnDiskPt.vcxproj b/contrib/other-builds/OnDiskPt.vcxproj index 5a9d1368b..827291e7d 100644 --- a/contrib/other-builds/OnDiskPt.vcxproj +++ b/contrib/other-builds/OnDiskPt.vcxproj @@ -69,7 +69,7 @@ Disabled - WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + WITH_THREADS;NO_PIPES;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -84,7 +84,7 @@ MaxSpeed true - WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + WITH_THREADS;NO_PIPES;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) MultiThreadedDLL true diff --git a/contrib/other-builds/OnDiskPt/.cproject b/contrib/other-builds/OnDiskPt/.cproject index 41f2a5141..472888f48 100644 --- a/contrib/other-builds/OnDiskPt/.cproject +++ b/contrib/other-builds/OnDiskPt/.cproject @@ -41,9 +41,13 @@ + @@ -128,4 +132,5 @@ + diff --git a/contrib/other-builds/fuzzy-match.xcodeproj/project.pbxproj b/contrib/other-builds/fuzzy-match.xcodeproj/project.pbxproj new file mode 100644 index 000000000..8abb9ae17 --- /dev/null +++ b/contrib/other-builds/fuzzy-match.xcodeproj/project.pbxproj @@ -0,0 +1,292 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 1E42EFB615BEFAEB00E937EB /* fuzzy-match2.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E42EFA515BEFABD00E937EB /* fuzzy-match2.cpp */; }; + 1E42EFB715BEFAEB00E937EB /* SuffixArray.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E806DCF15BED3D4001914A2 /* SuffixArray.cpp */; }; + 1E42EFB815BEFAEB00E937EB /* SuffixArray.h in Sources */ = {isa = PBXBuildFile; fileRef = 1E806DD015BED3D4001914A2 /* SuffixArray.h */; }; + 1E42EFB915BEFAEB00E937EB /* Vocabulary.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E806DCA15BED3AC001914A2 /* Vocabulary.cpp */; }; + 1E42EFBA15BEFAEB00E937EB /* Vocabulary.h in Sources */ = {isa = PBXBuildFile; fileRef = 1E806DCB15BED3AC001914A2 /* Vocabulary.h */; }; + 1E806DCC15BED3AC001914A2 /* Vocabulary.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E806DCA15BED3AC001914A2 /* Vocabulary.cpp */; }; + 1E806DD115BED3D4001914A2 /* SuffixArray.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E806DCF15BED3D4001914A2 /* SuffixArray.cpp */; }; + 1ECD60A815C15E28004172A4 /* Util.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1ECD60A515C15D3A004172A4 /* Util.cpp */; }; +/* End PBXBuildFile section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 1E42EFAA15BEFAD300E937EB /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; + 1ED87EEB15BED331003E47AA /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 1E42EFA515BEFABD00E937EB /* fuzzy-match2.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = "fuzzy-match2.cpp"; path = "../tm-mt-integration/fuzzy-match2.cpp"; sourceTree = ""; }; + 1E42EFAC15BEFAD300E937EB /* fuzzy-match2 */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = "fuzzy-match2"; sourceTree = BUILT_PRODUCTS_DIR; }; + 1E42EFD115C00AC100E937EB /* fuzzy-match2.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = "fuzzy-match2.h"; path = "../tm-mt-integration/fuzzy-match2.h"; sourceTree = ""; }; + 1E42EFD215C00BAE00E937EB /* Util.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Util.h; path = "../tm-mt-integration/Util.h"; sourceTree = ""; }; + 1E42EFD315C00C0A00E937EB /* SentenceAlignment.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SentenceAlignment.h; path = "../tm-mt-integration/SentenceAlignment.h"; sourceTree = ""; }; + 1E42EFD715C00D6300E937EB /* Match.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Match.h; path = "../tm-mt-integration/Match.h"; sourceTree = ""; }; + 1E806DCA15BED3AC001914A2 /* Vocabulary.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Vocabulary.cpp; path = "../tm-mt-integration/Vocabulary.cpp"; sourceTree = ""; }; + 1E806DCB15BED3AC001914A2 /* Vocabulary.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Vocabulary.h; path = "../tm-mt-integration/Vocabulary.h"; sourceTree = ""; }; + 1E806DCF15BED3D4001914A2 /* SuffixArray.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = SuffixArray.cpp; path = "../tm-mt-integration/SuffixArray.cpp"; sourceTree = ""; }; + 1E806DD015BED3D4001914A2 /* SuffixArray.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SuffixArray.h; path = "../tm-mt-integration/SuffixArray.h"; sourceTree = ""; }; + 1ECD60A515C15D3A004172A4 /* Util.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Util.cpp; path = "../tm-mt-integration/Util.cpp"; sourceTree = ""; }; + 1ED87EED15BED331003E47AA /* fuzzy-match */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = "fuzzy-match"; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 1E42EFA915BEFAD300E937EB /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 1ED87EEA15BED331003E47AA /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 1ED87EE215BED32F003E47AA = { + isa = PBXGroup; + children = ( + 1E42EFD715C00D6300E937EB /* Match.h */, + 1E42EFD315C00C0A00E937EB /* SentenceAlignment.h */, + 1E42EFD215C00BAE00E937EB /* Util.h */, + 1ECD60A515C15D3A004172A4 /* Util.cpp */, + 1E806DCF15BED3D4001914A2 /* SuffixArray.cpp */, + 1E806DD015BED3D4001914A2 /* SuffixArray.h */, + 1E42EFD115C00AC100E937EB /* fuzzy-match2.h */, + 1E42EFA515BEFABD00E937EB /* fuzzy-match2.cpp */, + 1E806DCA15BED3AC001914A2 /* Vocabulary.cpp */, + 1E806DCB15BED3AC001914A2 /* Vocabulary.h */, + 1ED87EEE15BED331003E47AA /* Products */, + ); + sourceTree = ""; + }; + 1ED87EEE15BED331003E47AA /* Products */ = { + isa = PBXGroup; + children = ( + 1ED87EED15BED331003E47AA /* fuzzy-match */, + 1E42EFAC15BEFAD300E937EB /* fuzzy-match2 */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 1E42EFAB15BEFAD300E937EB /* fuzzy-match2 */ = { + isa = PBXNativeTarget; + buildConfigurationList = 1E42EFB315BEFAD300E937EB /* Build configuration list for PBXNativeTarget "fuzzy-match2" */; + buildPhases = ( + 1E42EFA815BEFAD300E937EB /* Sources */, + 1E42EFA915BEFAD300E937EB /* Frameworks */, + 1E42EFAA15BEFAD300E937EB /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = "fuzzy-match2"; + productName = "fuzzy-match2"; + productReference = 1E42EFAC15BEFAD300E937EB /* fuzzy-match2 */; + productType = "com.apple.product-type.tool"; + }; + 1ED87EEC15BED331003E47AA /* fuzzy-match */ = { + isa = PBXNativeTarget; + buildConfigurationList = 1ED87EF715BED331003E47AA /* Build configuration list for PBXNativeTarget "fuzzy-match" */; + buildPhases = ( + 1ED87EE915BED331003E47AA /* Sources */, + 1ED87EEA15BED331003E47AA /* Frameworks */, + 1ED87EEB15BED331003E47AA /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = "fuzzy-match"; + productName = "fuzzy-match"; + productReference = 1ED87EED15BED331003E47AA /* fuzzy-match */; + productType = "com.apple.product-type.tool"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 1ED87EE415BED32F003E47AA /* Project object */ = { + isa = PBXProject; + buildConfigurationList = 1ED87EE715BED32F003E47AA /* Build configuration list for PBXProject "fuzzy-match" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + ); + mainGroup = 1ED87EE215BED32F003E47AA; + productRefGroup = 1ED87EEE15BED331003E47AA /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 1ED87EEC15BED331003E47AA /* fuzzy-match */, + 1E42EFAB15BEFAD300E937EB /* fuzzy-match2 */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + 1E42EFA815BEFAD300E937EB /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 1ECD60A815C15E28004172A4 /* Util.cpp in Sources */, + 1E42EFB615BEFAEB00E937EB /* fuzzy-match2.cpp in Sources */, + 1E42EFB715BEFAEB00E937EB /* SuffixArray.cpp in Sources */, + 1E42EFB815BEFAEB00E937EB /* SuffixArray.h in Sources */, + 1E42EFB915BEFAEB00E937EB /* Vocabulary.cpp in Sources */, + 1E42EFBA15BEFAEB00E937EB /* Vocabulary.h in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 1ED87EE915BED331003E47AA /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 1E806DCC15BED3AC001914A2 /* Vocabulary.cpp in Sources */, + 1E806DD115BED3D4001914A2 /* SuffixArray.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 1E42EFB415BEFAD300E937EB /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 1E42EFB515BEFAD300E937EB /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; + 1ED87EF515BED331003E47AA /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + COPY_PHASE_STRIP = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_SYMBOLS_PRIVATE_EXTERN = NO; + GCC_VERSION = com.apple.compilers.llvm.clang.1_0; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 10.7; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + }; + name = Debug; + }; + 1ED87EF615BED331003E47AA /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + COPY_PHASE_STRIP = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_VERSION = com.apple.compilers.llvm.clang.1_0; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 10.7; + SDKROOT = macosx; + }; + name = Release; + }; + 1ED87EF815BED331003E47AA /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 1ED87EF915BED331003E47AA /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 1E42EFB315BEFAD300E937EB /* Build configuration list for PBXNativeTarget "fuzzy-match2" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1E42EFB415BEFAD300E937EB /* Debug */, + 1E42EFB515BEFAD300E937EB /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 1ED87EE715BED32F003E47AA /* Build configuration list for PBXProject "fuzzy-match" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1ED87EF515BED331003E47AA /* Debug */, + 1ED87EF615BED331003E47AA /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 1ED87EF715BED331003E47AA /* Build configuration list for PBXNativeTarget "fuzzy-match" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1ED87EF815BED331003E47AA /* Debug */, + 1ED87EF915BED331003E47AA /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 1ED87EE415BED32F003E47AA /* Project object */; +} diff --git a/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcdebugger/Breakpoints.xcbkptlist b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcdebugger/Breakpoints.xcbkptlist new file mode 100644 index 000000000..cebcbdcb5 --- /dev/null +++ b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcdebugger/Breakpoints.xcbkptlist @@ -0,0 +1,21 @@ + + + + + + + diff --git a/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/fuzzy-match.xcscheme b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/fuzzy-match.xcscheme new file mode 100644 index 000000000..4ffb0bc96 --- /dev/null +++ b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/fuzzy-match.xcscheme @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/fuzzy-match2.xcscheme b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/fuzzy-match2.xcscheme new file mode 100644 index 000000000..124bfd4b2 --- /dev/null +++ b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/fuzzy-match2.xcscheme @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 000000000..8a9f26d81 --- /dev/null +++ b/contrib/other-builds/fuzzy-match.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,32 @@ + + + + + SchemeUserState + + fuzzy-match.xcscheme + + orderHint + 0 + + fuzzy-match2.xcscheme + + orderHint + 1 + + + SuppressBuildableAutocreation + + 1E42EFAB15BEFAD300E937EB + + primary + + + 1ED87EEC15BED331003E47AA + + primary + + + + + diff --git a/contrib/other-builds/kenlm.vcxproj b/contrib/other-builds/kenlm.vcxproj index f9cf6a850..544600117 100755 --- a/contrib/other-builds/kenlm.vcxproj +++ b/contrib/other-builds/kenlm.vcxproj @@ -1,4 +1,4 @@ - + @@ -123,7 +123,12 @@ - + + C:\Program Files\boost\boost_1_47;$(IncludePath) + + + C:\Program Files\boost\boost_1_47;$(IncludePath) + @@ -131,7 +136,7 @@ Level3 Disabled WITH_THREADS;NO_PIPES;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - $(SolutionDir)\..\..\lm\msinttypes;C:\boost\boost_1_47;$(SolutionDir)/../.. + C:\boost\boost_1_47;$(SolutionDir)/../.. Windows @@ -147,7 +152,7 @@ true true WITH_THREADS;NO_PIPES;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - $(SolutionDir)\..\..\lm\msinttypes;C:\boost\boost_1_47;$(SolutionDir)/../.. + C:\boost\boost_1_47;$(SolutionDir)/../.. MultiThreadedDLL diff --git a/contrib/other-builds/lm.xcodeproj/project.pbxproj b/contrib/other-builds/lm.xcodeproj/project.pbxproj index deb817b7c..2488f1439 100644 --- a/contrib/other-builds/lm.xcodeproj/project.pbxproj +++ b/contrib/other-builds/lm.xcodeproj/project.pbxproj @@ -405,6 +405,9 @@ /* Begin PBXProject section */ 1EE8C2E01476A48E002496F2 /* Project object */ = { isa = PBXProject; + attributes = { + LastUpgradeCheck = 0420; + }; buildConfigurationList = 1EE8C2E31476A48E002496F2 /* Build configuration list for PBXProject "lm" */; compatibilityVersion = "Xcode 3.2"; developmentRegion = English; @@ -539,6 +542,7 @@ isa = XCBuildConfiguration; buildSettings = { EXECUTABLE_PREFIX = lib; + GCC_PREPROCESSOR_DEFINITIONS = "KENLM_MAX_ORDER=7"; LIBRARY_SEARCH_PATHS = ( "$(inherited)", "\"$(SRCROOT)/../../lm/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi\"", @@ -556,6 +560,7 @@ isa = XCBuildConfiguration; buildSettings = { EXECUTABLE_PREFIX = lib; + GCC_PREPROCESSOR_DEFINITIONS = "KENLM_MAX_ORDER=7"; LIBRARY_SEARCH_PATHS = ( "$(inherited)", "\"$(SRCROOT)/../../lm/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi\"", diff --git a/contrib/other-builds/lm/.cproject b/contrib/other-builds/lm/.cproject index f89e80f49..8ecb60e02 100644 --- a/contrib/other-builds/lm/.cproject +++ b/contrib/other-builds/lm/.cproject @@ -42,7 +42,11 @@ + @@ -53,6 +57,9 @@ + + + @@ -122,4 +129,5 @@ + diff --git a/contrib/other-builds/lm/.project b/contrib/other-builds/lm/.project index 0d30e24cb..204771764 100644 --- a/contrib/other-builds/lm/.project +++ b/contrib/other-builds/lm/.project @@ -326,6 +326,21 @@ 1 PARENT-3-PROJECT_LOC/lm/trie_sort.hh + + value.hh + 1 + PARENT-3-PROJECT_LOC/lm/value.hh + + + value_build.cc + 1 + PARENT-3-PROJECT_LOC/lm/value_build.cc + + + value_build.hh + 1 + PARENT-3-PROJECT_LOC/lm/value_build.hh + virtual_interface.cc 1 diff --git a/contrib/other-builds/mert.xcodeproj/project.pbxproj b/contrib/other-builds/mert.xcodeproj/project.pbxproj index 4d82aa4b8..76879e58e 100644 --- a/contrib/other-builds/mert.xcodeproj/project.pbxproj +++ b/contrib/other-builds/mert.xcodeproj/project.pbxproj @@ -312,6 +312,7 @@ 1E1D826815AC640800FE42E9 /* Release */, ); defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; }; 1EB0AEFF1593A2180007E2A4 /* Build configuration list for PBXProject "mert" */ = { isa = XCConfigurationList; diff --git a/contrib/other-builds/mert.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/contrib/other-builds/mert.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 000000000..03c6b7b80 --- /dev/null +++ b/contrib/other-builds/mert.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/contrib/other-builds/mert.xcodeproj/project.xcworkspace/xcuserdata/hieuhoang.xcuserdatad/UserInterfaceState.xcuserstate b/contrib/other-builds/mert.xcodeproj/project.xcworkspace/xcuserdata/hieuhoang.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 000000000..eef05294a --- /dev/null +++ b/contrib/other-builds/mert.xcodeproj/project.xcworkspace/xcuserdata/hieuhoang.xcuserdatad/UserInterfaceState.xcuserstate @@ -0,0 +1,8628 @@ + + + + + $archiver + NSKeyedArchiver + $objects + + $null + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 2 + + + CF$UID + 3 + + + NS.objects + + + CF$UID + 4 + + + CF$UID + 177 + + + + A0ED48DA-D116-4801-AB51-861E1E3CE459 + IDEWorkspaceDocument + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 5 + + + CF$UID + 6 + + + CF$UID + 7 + + + CF$UID + 8 + + + CF$UID + 9 + + + CF$UID + 10 + + + CF$UID + 11 + + + CF$UID + 12 + + + NS.objects + + + CF$UID + 13 + + + CF$UID + 14 + + + CF$UID + 16 + + + CF$UID + 17 + + + CF$UID + 2 + + + CF$UID + 8 + + + CF$UID + 128 + + + CF$UID + 128 + + + + IDEWindowFrame + IDEOrderedWorkspaceTabControllers + IDEWindowInFullscreenMode + IDEWorkspaceTabController_47815CCD-573D-4957-A6D1-F7389545EB27 + IDEWorkspaceWindowControllerUniqueIdentifier + IDEActiveWorkspaceTabController + IDEWindowToolbarIsVisible + IDEWindowTabBarIsVisible + {{0, 58}, {1280, 720}} + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 8 + + + + + $classes + + NSArray + NSObject + + $classname + NSArray + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 18 + + + CF$UID + 19 + + + CF$UID + 20 + + + CF$UID + 21 + + + CF$UID + 22 + + + CF$UID + 23 + + + CF$UID + 24 + + + CF$UID + 25 + + + NS.objects + + + CF$UID + 26 + + + CF$UID + 128 + + + CF$UID + 47 + + + CF$UID + 138 + + + CF$UID + 145 + + + CF$UID + 167 + + + CF$UID + 16 + + + CF$UID + 176 + + + + IDEEditorArea + IDEShowNavigator + AssistantEditorsLayout + IDEWorkspaceTabControllerUtilityAreaSplitView + IDENavigatorArea + IDEWorkspaceTabControllerDesignAreaSplitView + IDEShowUtilities + IDETabLabel + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 27 + + + CF$UID + 28 + + + CF$UID + 29 + + + CF$UID + 30 + + + CF$UID + 31 + + + CF$UID + 32 + + + CF$UID + 33 + + + CF$UID + 34 + + + NS.objects + + + CF$UID + 35 + + + CF$UID + 57 + + + CF$UID + 98 + + + CF$UID + 128 + + + CF$UID + 47 + + + CF$UID + 129 + + + CF$UID + 137 + + + CF$UID + 16 + + + + layoutTree + IDEEditorMode_Standard + IDEEDitorArea_DebugArea + IDEShowEditor + EditorMode + DebuggerSplitView + DefaultPersistentRepresentations + ShowDebuggerArea + + $class + + CF$UID + 56 + + geniusEditorContextNode + + CF$UID + 0 + + primaryEditorContextNode + + CF$UID + 36 + + rootLayoutTreeNode + + CF$UID + 53 + + + + $class + + CF$UID + 55 + + children + + CF$UID + 0 + + contentType + 1 + documentArchivableRepresentation + + CF$UID + 37 + + orientation + 0 + parent + + CF$UID + 53 + + + + $class + + CF$UID + 52 + + DocumentLocation + + CF$UID + 48 + + DomainIdentifier + + CF$UID + 38 + + IdentifierPath + + CF$UID + 39 + + IndexOfDocumentIdentifier + + CF$UID + 47 + + + Xcode.IDENavigableItemDomain.WorkspaceStructure + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 40 + + + CF$UID + 43 + + + CF$UID + 45 + + + + + $class + + CF$UID + 42 + + Identifier + + CF$UID + 41 + + + InterpolatedScorer.h + + $classes + + IDEArchivableStringIndexPair + NSObject + + $classname + IDEArchivableStringIndexPair + + + $class + + CF$UID + 42 + + Identifier + + CF$UID + 44 + + + mert_lib.xcodeproj + + $class + + CF$UID + 42 + + Identifier + + CF$UID + 46 + + + mert + 0 + + $class + + CF$UID + 51 + + documentURL + + CF$UID + 49 + + timestamp + + CF$UID + 0 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/InterpolatedScorer.h + + + $classes + + NSMutableString + NSString + NSObject + + $classname + NSMutableString + + + $classes + + DVTDocumentLocation + NSObject + + $classname + DVTDocumentLocation + + + $classes + + IDENavigableItemArchivableRepresentation + NSObject + + $classname + IDENavigableItemArchivableRepresentation + + + $class + + CF$UID + 55 + + children + + CF$UID + 54 + + contentType + 0 + documentArchivableRepresentation + + CF$UID + 0 + + orientation + 0 + parent + + CF$UID + 0 + + + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 36 + + + + + $classes + + IDEWorkspaceTabControllerLayoutTreeNode + NSObject + + $classname + IDEWorkspaceTabControllerLayoutTreeNode + + + $classes + + IDEWorkspaceTabControllerLayoutTree + NSObject + + $classname + IDEWorkspaceTabControllerLayoutTree + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 58 + + + NS.objects + + + CF$UID + 59 + + + + EditorLayout_PersistentRepresentation + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 60 + + + NS.objects + + + CF$UID + 61 + + + + Main + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 62 + + + CF$UID + 63 + + + CF$UID + 64 + + + NS.objects + + + CF$UID + 65 + + + CF$UID + 47 + + + CF$UID + 96 + + + + EditorLayout_StateSavingStateDictionaries + EditorLayout_Selected + EditorLayout_Geometry + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 66 + + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 67 + + + CF$UID + 68 + + + CF$UID + 69 + + + CF$UID + 70 + + + CF$UID + 71 + + + CF$UID + 72 + + + CF$UID + 73 + + + NS.objects + + + CF$UID + 74 + + + CF$UID + 75 + + + CF$UID + 81 + + + CF$UID + 90 + + + CF$UID + 41 + + + CF$UID + 91 + + + CF$UID + 92 + + + + FileDataType + ArchivableRepresentation + EditorState + NavigableItemName + DocumentNavigableItemName + DocumentExtensionIdentifier + DocumentURL + public.c-header + + $class + + CF$UID + 52 + + DocumentLocation + + CF$UID + 48 + + DomainIdentifier + + CF$UID + 38 + + IdentifierPath + + CF$UID + 76 + + IndexOfDocumentIdentifier + + CF$UID + 47 + + + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 77 + + + CF$UID + 78 + + + CF$UID + 79 + + + + + $class + + CF$UID + 42 + + Identifier + + CF$UID + 41 + + + + $class + + CF$UID + 42 + + Identifier + + CF$UID + 44 + + + + $class + + CF$UID + 42 + + Identifier + + CF$UID + 80 + + + mert + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 82 + + + CF$UID + 83 + + + CF$UID + 84 + + + CF$UID + 85 + + + NS.objects + + + CF$UID + 86 + + + CF$UID + 87 + + + CF$UID + 16 + + + CF$UID + 88 + + + + PrimaryDocumentTimestamp + PrimaryDocumentVisibleCharacterRange + HideAllIssues + PrimaryDocumentSelectedCharacterRange + 363696391.20448101 + {0, 1309} + {332, 0} + + $classes + + NSDictionary + NSObject + + $classname + NSDictionary + + class InterpolatedScorer + Xcode.IDEKit.EditorDocument.SourceCode + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 93 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/InterpolatedScorer.h + + $classes + + NSURL + NSObject + + $classname + NSURL + + + $classes + + NSMutableDictionary + NSDictionary + NSObject + + $classname + NSMutableDictionary + + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 97 + + + + {{0, 0}, {1020, 622}} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 99 + + + CF$UID + 100 + + + CF$UID + 101 + + + CF$UID + 102 + + + CF$UID + 103 + + + CF$UID + 104 + + + NS.objects + + + CF$UID + 105 + + + CF$UID + 106 + + + CF$UID + 108 + + + CF$UID + 105 + + + CF$UID + 110 + + + CF$UID + 122 + + + + LayoutFocusMode + console + variables + LayoutMode + IDEDebuggerAreaSplitView + IDEDebugArea_SplitView + 1 + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 107 + + + NS.objects + + + CF$UID + 47 + + + + ConsoleFilterMode + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 109 + + + NS.objects + + + CF$UID + 105 + + + + VariablesViewSelectedScope + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 111 + + + NS.objects + + + CF$UID + 112 + + + + DVTSplitViewItems + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 113 + + + CF$UID + 118 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 116 + + + CF$UID + 117 + + + + DVTIdentifier + DVTViewMagnitude + VariablesView + 510 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 119 + + + CF$UID + 120 + + + + ConsoleArea + 509 + + $classes + + NSMutableArray + NSArray + NSObject + + $classname + NSMutableArray + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 111 + + + NS.objects + + + CF$UID + 123 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 124 + + + CF$UID + 126 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 116 + + + CF$UID + 125 + + + + 510 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 119 + + + CF$UID + 127 + + + + 509 + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 111 + + + NS.objects + + + CF$UID + 130 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 131 + + + CF$UID + 134 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 132 + + + CF$UID + 133 + + + + IDEEditor + 203 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 135 + + + CF$UID + 136 + + + + IDEDebuggerArea + 115 + + $class + + CF$UID + 95 + + NS.keys + + NS.objects + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 111 + + + NS.objects + + + CF$UID + 139 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 140 + + + CF$UID + 143 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 141 + + + CF$UID + 142 + + + + + 398 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 141 + + + CF$UID + 144 + + + + 224 + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 146 + + + CF$UID + 147 + + + NS.objects + + + CF$UID + 147 + + + CF$UID + 148 + + + + SelectedNavigator + Xcode.IDEKit.Navigator.Structure + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 149 + + + CF$UID + 150 + + + CF$UID + 151 + + + CF$UID + 152 + + + CF$UID + 153 + + + CF$UID + 154 + + + CF$UID + 155 + + + NS.objects + + + CF$UID + 156 + + + CF$UID + 16 + + + CF$UID + 157 + + + CF$UID + 16 + + + CF$UID + 16 + + + CF$UID + 159 + + + CF$UID + 162 + + + + IDEVisibleRect + IDEUnsavedDocumentFilteringEnabled + IDENavigatorExpandedItemsBeforeFilteringSet + IDERecentDocumentFilteringEnabled + IDESCMStatusFilteringEnabled + IDESelectedObjects + IDEExpandedItemsSet + {{0, 300}, {259, 578}} + + $class + + CF$UID + 158 + + NS.objects + + + + $classes + + NSSet + NSObject + + $classname + NSSet + + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 160 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 161 + + + CF$UID + 44 + + + CF$UID + 41 + + + + mert + + $class + + CF$UID + 158 + + NS.objects + + + CF$UID + 163 + + + CF$UID + 165 + + + CF$UID + 166 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 161 + + + CF$UID + 164 + + + + Products + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 161 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 161 + + + CF$UID + 44 + + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 111 + + + NS.objects + + + CF$UID + 168 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 169 + + + CF$UID + 171 + + + CF$UID + 173 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 22 + + + CF$UID + 170 + + + + 260 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 18 + + + CF$UID + 172 + + + + 1020 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 114 + + + CF$UID + 115 + + + NS.objects + + + CF$UID + 174 + + + CF$UID + 175 + + + + IDEUtilitiesArea + 260 + InterpolatedScorer.h + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 178 + + + CF$UID + 179 + + + CF$UID + 180 + + + CF$UID + 181 + + + CF$UID + 182 + + + CF$UID + 183 + + + CF$UID + 184 + + + CF$UID + 185 + + + CF$UID + 186 + + + CF$UID + 187 + + + NS.objects + + + CF$UID + 16 + + + CF$UID + 188 + + + CF$UID + 47 + + + CF$UID + 655 + + + CF$UID + 660 + + + CF$UID + 663 + + + CF$UID + 694 + + + CF$UID + 695 + + + CF$UID + 16 + + + CF$UID + 16 + + + + BreakpointsActivated + DefaultEditorStatesForURLs + DebuggingWindowBehavior + ActiveRunDestination + ActiveScheme + LastCompletedPersistentSchemeBasedActivityReport + DocumentWindows + RecentEditorDocumentURLs + AppFocusInMiniDebugging + MiniDebuggingConsole + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 189 + + + CF$UID + 190 + + + NS.objects + + + CF$UID + 191 + + + CF$UID + 613 + + + + Xcode.Xcode3ProjectSupport.EditorDocument.Xcode3Project + Xcode.IDEKit.EditorDocument.SourceCode + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 192 + + + CF$UID + 194 + + + NS.objects + + + CF$UID + 196 + + + CF$UID + 414 + + + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 193 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/contrib/other-builds/mert.xcodeproj/ + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 195 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/contrib/other-builds/mert_lib.xcodeproj/ + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 197 + + + CF$UID + 198 + + + CF$UID + 199 + + + CF$UID + 200 + + + NS.objects + + + CF$UID + 201 + + + CF$UID + 211 + + + CF$UID + 212 + + + CF$UID + 413 + + + + Xcode3ProjectEditor.sourceList.splitview + Xcode3ProjectEditorPreviousTargetEditorClass + Xcode3ProjectEditorSelectedDocumentLocations + Xcode3ProjectEditor_Xcode3BuildSettingsEditor + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 202 + + + NS.objects + + + CF$UID + 203 + + + + DVTSplitViewItems + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 204 + + + CF$UID + 209 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 205 + + + CF$UID + 206 + + + NS.objects + + + CF$UID + 207 + + + CF$UID + 208 + + + + DVTIdentifier + DVTViewMagnitude + + 162 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 205 + + + CF$UID + 206 + + + NS.objects + + + CF$UID + 207 + + + CF$UID + 210 + + + + 858 + Xcode3BuildSettingsEditor + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 213 + + + + + $class + + CF$UID + 412 + + documentURL + + CF$UID + 214 + + selection + + CF$UID + 216 + + timestamp + + CF$UID + 215 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/contrib/other-builds/mert.xcodeproj/ + 363627943.189156 + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 217 + + + CF$UID + 218 + + + CF$UID + 219 + + + NS.objects + + + CF$UID + 211 + + + CF$UID + 220 + + + CF$UID + 221 + + + + Editor + Target + Xcode3BuildSettingsEditorLocations + mert + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 222 + + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 223 + + + CF$UID + 224 + + + CF$UID + 225 + + + CF$UID + 226 + + + CF$UID + 227 + + + CF$UID + 228 + + + NS.objects + + + CF$UID + 229 + + + CF$UID + 230 + + + CF$UID + 229 + + + CF$UID + 229 + + + CF$UID + 231 + + + CF$UID + 105 + + + + Xcode3BuildSettingsEditorMode + Selected Build Properties + Xcode3BuildSettingsEditorDisplayMode + Xcode3BuildPropertyValueDisplayMode + Collapsed Build Property Categories + Xcode3BuildPropertyNameDisplayMode + 0 + + $class + + CF$UID + 121 + + NS.objects + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 232 + + + CF$UID + 233 + + + CF$UID + 234 + + + CF$UID + 235 + + + CF$UID + 236 + + + CF$UID + 237 + + + CF$UID + 238 + + + CF$UID + 239 + + + CF$UID + 240 + + + CF$UID + 241 + + + CF$UID + 242 + + + CF$UID + 243 + + + CF$UID + 244 + + + CF$UID + 245 + + + CF$UID + 246 + + + CF$UID + 247 + + + CF$UID + 248 + + + CF$UID + 249 + + + CF$UID + 250 + + + CF$UID + 251 + + + CF$UID + 252 + + + CF$UID + 253 + + + CF$UID + 254 + + + CF$UID + 255 + + + CF$UID + 256 + + + CF$UID + 257 + + + CF$UID + 258 + + + CF$UID + 259 + + + CF$UID + 260 + + + CF$UID + 261 + + + CF$UID + 262 + + + CF$UID + 263 + + + CF$UID + 264 + + + CF$UID + 265 + + + CF$UID + 266 + + + CF$UID + 267 + + + CF$UID + 268 + + + CF$UID + 269 + + + CF$UID + 270 + + + CF$UID + 271 + + + CF$UID + 272 + + + CF$UID + 273 + + + CF$UID + 274 + + + CF$UID + 275 + + + CF$UID + 276 + + + CF$UID + 277 + + + CF$UID + 278 + + + CF$UID + 279 + + + CF$UID + 280 + + + CF$UID + 281 + + + CF$UID + 282 + + + CF$UID + 283 + + + CF$UID + 284 + + + CF$UID + 285 + + + CF$UID + 286 + + + CF$UID + 287 + + + CF$UID + 288 + + + CF$UID + 289 + + + CF$UID + 290 + + + CF$UID + 291 + + + CF$UID + 292 + + + CF$UID + 293 + + + CF$UID + 294 + + + CF$UID + 295 + + + CF$UID + 296 + + + CF$UID + 297 + + + CF$UID + 298 + + + CF$UID + 299 + + + CF$UID + 300 + + + CF$UID + 301 + + + CF$UID + 302 + + + CF$UID + 303 + + + CF$UID + 304 + + + CF$UID + 305 + + + CF$UID + 306 + + + CF$UID + 307 + + + CF$UID + 308 + + + CF$UID + 309 + + + CF$UID + 310 + + + CF$UID + 311 + + + CF$UID + 312 + + + CF$UID + 313 + + + CF$UID + 314 + + + CF$UID + 315 + + + CF$UID + 316 + + + CF$UID + 317 + + + CF$UID + 318 + + + CF$UID + 319 + + + CF$UID + 320 + + + CF$UID + 321 + + + CF$UID + 322 + + + CF$UID + 323 + + + CF$UID + 324 + + + CF$UID + 325 + + + CF$UID + 326 + + + CF$UID + 327 + + + CF$UID + 328 + + + CF$UID + 329 + + + CF$UID + 330 + + + CF$UID + 331 + + + CF$UID + 332 + + + CF$UID + 333 + + + CF$UID + 334 + + + CF$UID + 335 + + + CF$UID + 336 + + + CF$UID + 337 + + + CF$UID + 338 + + + CF$UID + 339 + + + CF$UID + 340 + + + CF$UID + 341 + + + CF$UID + 342 + + + CF$UID + 343 + + + CF$UID + 344 + + + CF$UID + 345 + + + CF$UID + 346 + + + CF$UID + 347 + + + CF$UID + 348 + + + CF$UID + 349 + + + CF$UID + 350 + + + CF$UID + 351 + + + CF$UID + 352 + + + CF$UID + 353 + + + CF$UID + 354 + + + CF$UID + 355 + + + CF$UID + 356 + + + CF$UID + 357 + + + CF$UID + 358 + + + CF$UID + 359 + + + CF$UID + 360 + + + CF$UID + 361 + + + CF$UID + 362 + + + CF$UID + 363 + + + CF$UID + 364 + + + CF$UID + 365 + + + CF$UID + 366 + + + CF$UID + 367 + + + CF$UID + 368 + + + CF$UID + 369 + + + CF$UID + 370 + + + CF$UID + 371 + + + CF$UID + 372 + + + CF$UID + 373 + + + CF$UID + 374 + + + CF$UID + 375 + + + CF$UID + 376 + + + CF$UID + 377 + + + CF$UID + 378 + + + CF$UID + 379 + + + CF$UID + 380 + + + CF$UID + 381 + + + CF$UID + 382 + + + CF$UID + 383 + + + CF$UID + 384 + + + CF$UID + 385 + + + CF$UID + 386 + + + CF$UID + 387 + + + CF$UID + 388 + + + CF$UID + 389 + + + CF$UID + 390 + + + CF$UID + 391 + + + CF$UID + 392 + + + CF$UID + 393 + + + CF$UID + 394 + + + CF$UID + 395 + + + CF$UID + 396 + + + CF$UID + 397 + + + CF$UID + 398 + + + CF$UID + 399 + + + CF$UID + 400 + + + CF$UID + 401 + + + CF$UID + 402 + + + CF$UID + 403 + + + CF$UID + 404 + + + CF$UID + 405 + + + CF$UID + 406 + + + CF$UID + 407 + + + CF$UID + 408 + + + CF$UID + 409 + + + CF$UID + 410 + + + CF$UID + 411 + + + + + $class + + CF$UID + 50 + + NS.string + Architectures||ADDITIONAL_SDKS + + + $class + + CF$UID + 50 + + NS.string + Architectures||ARCHS + + + $class + + CF$UID + 50 + + NS.string + Architectures||SDKROOT + + + $class + + CF$UID + 50 + + NS.string + Architectures||SUPPORTED_PLATFORMS + + + $class + + CF$UID + 50 + + NS.string + Architectures||VALID_ARCHS + + + $class + + CF$UID + 50 + + NS.string + Build Locations||SYMROOT + + + $class + + CF$UID + 50 + + NS.string + Build Locations||OBJROOT + + + $class + + CF$UID + 50 + + NS.string + Build Locations||SHARED_PRECOMPS_DIR + + + $class + + CF$UID + 50 + + NS.string + Build Options||BUILD_VARIANTS + + + $class + + CF$UID + 50 + + NS.string + Build Options||GCC_VERSION + + + $class + + CF$UID + 50 + + NS.string + Build Options||ENABLE_OPENMP_SUPPORT + + + $class + + CF$UID + 50 + + NS.string + Build Options||GENERATE_PROFILING_CODE + + + $class + + CF$UID + 50 + + NS.string + Build Options||PRECOMPS_INCLUDE_HEADERS_FROM_BUILT_PRODUCTS_DIR + + + $class + + CF$UID + 50 + + NS.string + Build Options||RUN_CLANG_STATIC_ANALYZER + + + $class + + CF$UID + 50 + + NS.string + Build Options||SCAN_ALL_SOURCE_FILES_FOR_INCLUDES + + + $class + + CF$UID + 50 + + NS.string + Build Options||VALIDATE_PRODUCT + + + $class + + CF$UID + 50 + + NS.string + Code Signing||CODE_SIGN_ENTITLEMENTS + + + $class + + CF$UID + 50 + + NS.string + Code Signing||CODE_SIGN_IDENTITY + + + $class + + CF$UID + 50 + + NS.string + Code Signing||CODE_SIGN_RESOURCE_RULES_PATH + + + $class + + CF$UID + 50 + + NS.string + Code Signing||OTHER_CODE_SIGN_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Deployment||STRIPFLAGS + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_GROUP + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_OWNER + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_MODE + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_PERMISSIONS_FILES + + + $class + + CF$UID + 50 + + NS.string + Deployment||COMBINE_HIDPI_IMAGES + + + $class + + CF$UID + 50 + + NS.string + Deployment||DEPLOYMENT_LOCATION + + + $class + + CF$UID + 50 + + NS.string + Deployment||DEPLOYMENT_POSTPROCESSING + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_GROUP + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_OWNER + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_MODE_FLAG + + + $class + + CF$UID + 50 + + NS.string + Deployment||DSTROOT + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_PATH + + + $class + + CF$UID + 50 + + NS.string + Deployment||MACOSX_DEPLOYMENT_TARGET + + + $class + + CF$UID + 50 + + NS.string + Deployment||SKIP_INSTALL + + + $class + + CF$UID + 50 + + NS.string + Deployment||STRIP_INSTALLED_PRODUCT + + + $class + + CF$UID + 50 + + NS.string + Deployment||STRIP_STYLE + + + $class + + CF$UID + 50 + + NS.string + Deployment||SEPARATE_STRIP + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_NAME + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_START + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_STOP + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_VERSION + + + $class + + CF$UID + 50 + + NS.string + Linking||BUNDLE_LOADER + + + $class + + CF$UID + 50 + + NS.string + Linking||DYLIB_COMPATIBILITY_VERSION + + + $class + + CF$UID + 50 + + NS.string + Linking||DYLIB_CURRENT_VERSION + + + $class + + CF$UID + 50 + + NS.string + Linking||DEAD_CODE_STRIPPING + + + $class + + CF$UID + 50 + + NS.string + Linking||LINKER_DISPLAYS_MANGLED_NAMES + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_NO_PIE + + + $class + + CF$UID + 50 + + NS.string + Linking||PRESERVE_DEAD_CODE_INITS_AND_TERMS + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_DYLIB_INSTALL_NAME + + + $class + + CF$UID + 50 + + NS.string + Linking||EXPORTED_SYMBOLS_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||INIT_ROUTINE + + + $class + + CF$UID + 50 + + NS.string + Linking||LINK_WITH_STANDARD_LIBRARIES + + + $class + + CF$UID + 50 + + NS.string + Linking||MACH_O_TYPE + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_OPENMP_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||ORDER_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||OTHER_LDFLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||GENERATE_MASTER_OBJECT_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||PRELINK_LIBS + + + $class + + CF$UID + 50 + + NS.string + Linking||KEEP_PRIVATE_EXTERNS + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_RUNPATH_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Linking||SEPARATE_SYMBOL_EDIT + + + $class + + CF$UID + 50 + + NS.string + Linking||PRELINK_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||SECTORDER_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||UNEXPORTED_SYMBOLS_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||WARNING_LDFLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_GENERATE_MAP_FILE + + + $class + + CF$UID + 50 + + NS.string + Packaging||APPLY_RULES_IN_COPY_FILES + + + $class + + CF$UID + 50 + + NS.string + Packaging||EXECUTABLE_EXTENSION + + + $class + + CF$UID + 50 + + NS.string + Packaging||EXECUTABLE_PREFIX + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_EXPAND_BUILD_SETTINGS + + + $class + + CF$UID + 50 + + NS.string + Packaging||GENERATE_PKGINFO_FILE + + + $class + + CF$UID + 50 + + NS.string + Packaging||FRAMEWORK_VERSION + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_FILE + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_OTHER_PREPROCESSOR_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_OUTPUT_FORMAT + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_PREPROCESSOR_DEFINITIONS + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_PREFIX_HEADER + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_PREPROCESS + + + $class + + CF$UID + 50 + + NS.string + Packaging||COPYING_PRESERVES_HFS_DATA + + + $class + + CF$UID + 50 + + NS.string + Packaging||PRIVATE_HEADERS_FOLDER_PATH + + + $class + + CF$UID + 50 + + NS.string + Packaging||PRODUCT_NAME + + + $class + + CF$UID + 50 + + NS.string + Packaging||PLIST_FILE_OUTPUT_FORMAT + + + $class + + CF$UID + 50 + + NS.string + Packaging||PUBLIC_HEADERS_FOLDER_PATH + + + $class + + CF$UID + 50 + + NS.string + Packaging||STRINGS_FILE_OUTPUT_ENCODING + + + $class + + CF$UID + 50 + + NS.string + Packaging||WRAPPER_EXTENSION + + + $class + + CF$UID + 50 + + NS.string + Search Paths||ALWAYS_SEARCH_USER_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||FRAMEWORK_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||HEADER_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||LIBRARY_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||REZ_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||EXCLUDED_RECURSIVE_SEARCH_PATH_SUBDIRECTORIES + + + $class + + CF$UID + 50 + + NS.string + Search Paths||INCLUDED_RECURSIVE_SEARCH_PATH_SUBDIRECTORIES + + + $class + + CF$UID + 50 + + NS.string + Search Paths||USER_HEADER_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||OTHER_TEST_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||TEST_AFTER_BUILD + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||TEST_HOST + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||TEST_RIG + + + $class + + CF$UID + 50 + + NS.string + Versioning||CURRENT_PROJECT_VERSION + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_FILE + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_EXPORT_DECL + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_PREFIX + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_SUFFIX + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSIONING_SYSTEM + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_BUILDER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_FAST_OBJC_DISPATCH + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SSE3_EXTENSIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SSE41_EXTENSIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SSE42_EXTENSIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SUPPLEMENTAL_SSE3_INSTRUCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_STRICT_ALIASING + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_GENERATE_DEBUGGING_SYMBOLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_DYNAMIC_NO_PIC + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_INLINES_ARE_PRIVATE_EXTERN + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_KERNEL_DEVELOPMENT + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||LLVM_LTO + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_REUSE_STRINGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_NO_COMMON_BLOCKS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_OBJC_GC + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_FAST_MATH + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_THREADSAFE_STATICS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_UNROLL_LOOPS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_CHAR_IS_UNSIGNED_CHAR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_ASM_KEYWORD + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_C_LANGUAGE_STANDARD + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_INPUT_FILETYPE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_CPP_EXCEPTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_CPP_RTTI + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_LINK_WITH_DYNAMIC_LIBRARIES + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_OBJC_EXCEPTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_TRIGRAPHS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_FLOATING_POINT_LIBRARY_CALLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_USE_INDIRECT_FUNCTION_CALLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_USE_REGISTER_FUNCTION_CALLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_INCREASE_PRECOMPILED_HEADER_SHARING + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_CW_ASM_SYNTAX + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||OTHER_CFLAGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||OTHER_CPLUSPLUSFLAGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_PRECOMPILE_PREFIX_HEADER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_PREFIX_HEADER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_BUILTIN_FUNCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_PASCAL_STRINGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_FORCE_CPU_SUBTYPE_ALL + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_SHORT_ENUMS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_USE_STANDARD_INCLUDE_SEARCHING + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Preprocessing||GCC_PREPROCESSOR_DEFINITIONS_NOT_USED_IN_PRECOMPS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_CHECK_SWITCH_STATEMENTS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_FOUR_CHARACTER_CONSTANTS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_SHADOW + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_64_TO_32_BIT_CONVERSION + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ALLOW_INCOMPLETE_PROTOCOL + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_INHIBIT_ALL_WARNINGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_INITIALIZER_NOT_FULLY_BRACKETED + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_RETURN_TYPE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_MISSING_PARENTHESES + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_MISSING_FIELD_INITIALIZERS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_MISSING_PROTOTYPES + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_MISSING_NEWLINE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_MULTIPLE_DEFINITION_TYPES_FOR_SELECTOR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_NON_VIRTUAL_DESTRUCTOR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||WARNING_CFLAGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_HIDDEN_VIRTUAL_FUNCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_PEDANTIC + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_POINTER_SIGNEDNESS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_SIGN_COMPARE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_STRICT_SELECTOR_MATCH + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_TREAT_INCOMPATIBLE_POINTER_TYPE_WARNINGS_AS_ERRORS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_TREAT_IMPLICIT_FUNCTION_DECLARATIONS_AS_ERRORS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_TREAT_WARNINGS_AS_ERRORS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_TYPECHECK_CALLS_TO_PRINTF + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNDECLARED_SELECTOR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNINITIALIZED_AUTOS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNKNOWN_PRAGMAS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_FUNCTION + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_LABEL + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_PARAMETER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_VALUE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_VARIABLE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_DEPRECATED_FUNCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_INVALID_OFFSETOF_MACRO + + + $classes + + Xcode3ProjectDocumentLocation + DVTDocumentLocation + NSObject + + $classname + Xcode3ProjectDocumentLocation + + + $class + + CF$UID + 95 + + NS.keys + + NS.objects + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 197 + + + CF$UID + 198 + + + CF$UID + 199 + + + CF$UID + 200 + + + NS.objects + + + CF$UID + 415 + + + CF$UID + 211 + + + CF$UID + 421 + + + CF$UID + 612 + + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 202 + + + NS.objects + + + CF$UID + 416 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 417 + + + CF$UID + 419 + + + + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 205 + + + CF$UID + 206 + + + NS.objects + + + CF$UID + 207 + + + CF$UID + 418 + + + + 170 + + $class + + CF$UID + 89 + + NS.keys + + + CF$UID + 205 + + + CF$UID + 206 + + + NS.objects + + + CF$UID + 207 + + + CF$UID + 420 + + + + 850 + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 422 + + + + + $class + + CF$UID + 412 + + documentURL + + CF$UID + 423 + + selection + + CF$UID + 425 + + timestamp + + CF$UID + 424 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/contrib/other-builds/mert_lib.xcodeproj/ + 363694729.26263899 + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 217 + + + CF$UID + 218 + + + CF$UID + 219 + + + NS.objects + + + CF$UID + 211 + + + CF$UID + 426 + + + CF$UID + 427 + + + + mert_lib + + $class + + CF$UID + 15 + + NS.objects + + + CF$UID + 428 + + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 228 + + + CF$UID + 224 + + + CF$UID + 225 + + + CF$UID + 226 + + + CF$UID + 227 + + + CF$UID + 223 + + + NS.objects + + + CF$UID + 105 + + + CF$UID + 429 + + + CF$UID + 229 + + + CF$UID + 229 + + + CF$UID + 430 + + + CF$UID + 229 + + + + + $class + + CF$UID + 121 + + NS.objects + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 431 + + + CF$UID + 432 + + + CF$UID + 433 + + + CF$UID + 434 + + + CF$UID + 435 + + + CF$UID + 436 + + + CF$UID + 437 + + + CF$UID + 438 + + + CF$UID + 439 + + + CF$UID + 440 + + + CF$UID + 441 + + + CF$UID + 442 + + + CF$UID + 443 + + + CF$UID + 444 + + + CF$UID + 445 + + + CF$UID + 446 + + + CF$UID + 447 + + + CF$UID + 448 + + + CF$UID + 449 + + + CF$UID + 450 + + + CF$UID + 451 + + + CF$UID + 452 + + + CF$UID + 453 + + + CF$UID + 454 + + + CF$UID + 455 + + + CF$UID + 456 + + + CF$UID + 457 + + + CF$UID + 458 + + + CF$UID + 459 + + + CF$UID + 460 + + + CF$UID + 461 + + + CF$UID + 462 + + + CF$UID + 463 + + + CF$UID + 464 + + + CF$UID + 465 + + + CF$UID + 466 + + + CF$UID + 467 + + + CF$UID + 468 + + + CF$UID + 469 + + + CF$UID + 470 + + + CF$UID + 471 + + + CF$UID + 472 + + + CF$UID + 473 + + + CF$UID + 474 + + + CF$UID + 475 + + + CF$UID + 476 + + + CF$UID + 477 + + + CF$UID + 478 + + + CF$UID + 479 + + + CF$UID + 480 + + + CF$UID + 481 + + + CF$UID + 482 + + + CF$UID + 483 + + + CF$UID + 484 + + + CF$UID + 485 + + + CF$UID + 486 + + + CF$UID + 487 + + + CF$UID + 488 + + + CF$UID + 489 + + + CF$UID + 490 + + + CF$UID + 491 + + + CF$UID + 492 + + + CF$UID + 493 + + + CF$UID + 494 + + + CF$UID + 495 + + + CF$UID + 496 + + + CF$UID + 497 + + + CF$UID + 498 + + + CF$UID + 499 + + + CF$UID + 500 + + + CF$UID + 501 + + + CF$UID + 502 + + + CF$UID + 503 + + + CF$UID + 504 + + + CF$UID + 505 + + + CF$UID + 506 + + + CF$UID + 507 + + + CF$UID + 508 + + + CF$UID + 509 + + + CF$UID + 510 + + + CF$UID + 511 + + + CF$UID + 512 + + + CF$UID + 513 + + + CF$UID + 514 + + + CF$UID + 515 + + + CF$UID + 516 + + + CF$UID + 517 + + + CF$UID + 518 + + + CF$UID + 519 + + + CF$UID + 520 + + + CF$UID + 521 + + + CF$UID + 522 + + + CF$UID + 523 + + + CF$UID + 524 + + + CF$UID + 525 + + + CF$UID + 526 + + + CF$UID + 527 + + + CF$UID + 528 + + + CF$UID + 529 + + + CF$UID + 530 + + + CF$UID + 531 + + + CF$UID + 532 + + + CF$UID + 533 + + + CF$UID + 534 + + + CF$UID + 535 + + + CF$UID + 536 + + + CF$UID + 537 + + + CF$UID + 538 + + + CF$UID + 539 + + + CF$UID + 540 + + + CF$UID + 541 + + + CF$UID + 542 + + + CF$UID + 543 + + + CF$UID + 544 + + + CF$UID + 545 + + + CF$UID + 546 + + + CF$UID + 547 + + + CF$UID + 548 + + + CF$UID + 549 + + + CF$UID + 550 + + + CF$UID + 551 + + + CF$UID + 552 + + + CF$UID + 553 + + + CF$UID + 554 + + + CF$UID + 555 + + + CF$UID + 556 + + + CF$UID + 557 + + + CF$UID + 558 + + + CF$UID + 559 + + + CF$UID + 560 + + + CF$UID + 561 + + + CF$UID + 562 + + + CF$UID + 563 + + + CF$UID + 564 + + + CF$UID + 565 + + + CF$UID + 566 + + + CF$UID + 567 + + + CF$UID + 568 + + + CF$UID + 569 + + + CF$UID + 570 + + + CF$UID + 571 + + + CF$UID + 572 + + + CF$UID + 573 + + + CF$UID + 574 + + + CF$UID + 575 + + + CF$UID + 576 + + + CF$UID + 577 + + + CF$UID + 578 + + + CF$UID + 579 + + + CF$UID + 580 + + + CF$UID + 581 + + + CF$UID + 582 + + + CF$UID + 583 + + + CF$UID + 584 + + + CF$UID + 585 + + + CF$UID + 586 + + + CF$UID + 587 + + + CF$UID + 588 + + + CF$UID + 589 + + + CF$UID + 590 + + + CF$UID + 591 + + + CF$UID + 592 + + + CF$UID + 593 + + + CF$UID + 594 + + + CF$UID + 595 + + + CF$UID + 596 + + + CF$UID + 597 + + + CF$UID + 598 + + + CF$UID + 599 + + + CF$UID + 600 + + + CF$UID + 601 + + + CF$UID + 602 + + + CF$UID + 603 + + + CF$UID + 604 + + + CF$UID + 605 + + + CF$UID + 606 + + + CF$UID + 607 + + + CF$UID + 608 + + + CF$UID + 609 + + + CF$UID + 610 + + + CF$UID + 611 + + + + + $class + + CF$UID + 50 + + NS.string + Architectures||ADDITIONAL_SDKS + + + $class + + CF$UID + 50 + + NS.string + Architectures||ARCHS + + + $class + + CF$UID + 50 + + NS.string + Architectures||SDKROOT + + + $class + + CF$UID + 50 + + NS.string + Architectures||SUPPORTED_PLATFORMS + + + $class + + CF$UID + 50 + + NS.string + Architectures||VALID_ARCHS + + + $class + + CF$UID + 50 + + NS.string + Build Locations||SYMROOT + + + $class + + CF$UID + 50 + + NS.string + Build Locations||OBJROOT + + + $class + + CF$UID + 50 + + NS.string + Build Locations||SHARED_PRECOMPS_DIR + + + $class + + CF$UID + 50 + + NS.string + Build Options||BUILD_VARIANTS + + + $class + + CF$UID + 50 + + NS.string + Build Options||GCC_VERSION + + + $class + + CF$UID + 50 + + NS.string + Build Options||ENABLE_OPENMP_SUPPORT + + + $class + + CF$UID + 50 + + NS.string + Build Options||GENERATE_PROFILING_CODE + + + $class + + CF$UID + 50 + + NS.string + Build Options||PRECOMPS_INCLUDE_HEADERS_FROM_BUILT_PRODUCTS_DIR + + + $class + + CF$UID + 50 + + NS.string + Build Options||RUN_CLANG_STATIC_ANALYZER + + + $class + + CF$UID + 50 + + NS.string + Build Options||SCAN_ALL_SOURCE_FILES_FOR_INCLUDES + + + $class + + CF$UID + 50 + + NS.string + Build Options||VALIDATE_PRODUCT + + + $class + + CF$UID + 50 + + NS.string + Code Signing||CODE_SIGN_ENTITLEMENTS + + + $class + + CF$UID + 50 + + NS.string + Code Signing||CODE_SIGN_IDENTITY + + + $class + + CF$UID + 50 + + NS.string + Code Signing||CODE_SIGN_RESOURCE_RULES_PATH + + + $class + + CF$UID + 50 + + NS.string + Code Signing||OTHER_CODE_SIGN_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Deployment||STRIPFLAGS + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_GROUP + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_OWNER + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_MODE + + + $class + + CF$UID + 50 + + NS.string + Deployment||ALTERNATE_PERMISSIONS_FILES + + + $class + + CF$UID + 50 + + NS.string + Deployment||COMBINE_HIDPI_IMAGES + + + $class + + CF$UID + 50 + + NS.string + Deployment||DEPLOYMENT_LOCATION + + + $class + + CF$UID + 50 + + NS.string + Deployment||DEPLOYMENT_POSTPROCESSING + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_GROUP + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_OWNER + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_MODE_FLAG + + + $class + + CF$UID + 50 + + NS.string + Deployment||DSTROOT + + + $class + + CF$UID + 50 + + NS.string + Deployment||INSTALL_PATH + + + $class + + CF$UID + 50 + + NS.string + Deployment||MACOSX_DEPLOYMENT_TARGET + + + $class + + CF$UID + 50 + + NS.string + Deployment||SKIP_INSTALL + + + $class + + CF$UID + 50 + + NS.string + Deployment||STRIP_INSTALLED_PRODUCT + + + $class + + CF$UID + 50 + + NS.string + Deployment||STRIP_STYLE + + + $class + + CF$UID + 50 + + NS.string + Deployment||SEPARATE_STRIP + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_NAME + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_START + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_STOP + + + $class + + CF$UID + 50 + + NS.string + Kernel Module||MODULE_VERSION + + + $class + + CF$UID + 50 + + NS.string + Linking||BUNDLE_LOADER + + + $class + + CF$UID + 50 + + NS.string + Linking||DYLIB_COMPATIBILITY_VERSION + + + $class + + CF$UID + 50 + + NS.string + Linking||DYLIB_CURRENT_VERSION + + + $class + + CF$UID + 50 + + NS.string + Linking||DEAD_CODE_STRIPPING + + + $class + + CF$UID + 50 + + NS.string + Linking||LINKER_DISPLAYS_MANGLED_NAMES + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_NO_PIE + + + $class + + CF$UID + 50 + + NS.string + Linking||PRESERVE_DEAD_CODE_INITS_AND_TERMS + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_DYLIB_INSTALL_NAME + + + $class + + CF$UID + 50 + + NS.string + Linking||EXPORTED_SYMBOLS_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||INIT_ROUTINE + + + $class + + CF$UID + 50 + + NS.string + Linking||LINK_WITH_STANDARD_LIBRARIES + + + $class + + CF$UID + 50 + + NS.string + Linking||MACH_O_TYPE + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_OPENMP_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||ORDER_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||OTHER_LDFLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||GENERATE_MASTER_OBJECT_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||PRELINK_LIBS + + + $class + + CF$UID + 50 + + NS.string + Linking||KEEP_PRIVATE_EXTERNS + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_RUNPATH_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Linking||SEPARATE_SYMBOL_EDIT + + + $class + + CF$UID + 50 + + NS.string + Linking||PRELINK_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||SECTORDER_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||UNEXPORTED_SYMBOLS_FILE + + + $class + + CF$UID + 50 + + NS.string + Linking||WARNING_LDFLAGS + + + $class + + CF$UID + 50 + + NS.string + Linking||LD_GENERATE_MAP_FILE + + + $class + + CF$UID + 50 + + NS.string + Packaging||APPLY_RULES_IN_COPY_FILES + + + $class + + CF$UID + 50 + + NS.string + Packaging||EXECUTABLE_EXTENSION + + + $class + + CF$UID + 50 + + NS.string + Packaging||EXECUTABLE_PREFIX + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_EXPAND_BUILD_SETTINGS + + + $class + + CF$UID + 50 + + NS.string + Packaging||GENERATE_PKGINFO_FILE + + + $class + + CF$UID + 50 + + NS.string + Packaging||FRAMEWORK_VERSION + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_FILE + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_OTHER_PREPROCESSOR_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_OUTPUT_FORMAT + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_PREPROCESSOR_DEFINITIONS + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_PREFIX_HEADER + + + $class + + CF$UID + 50 + + NS.string + Packaging||INFOPLIST_PREPROCESS + + + $class + + CF$UID + 50 + + NS.string + Packaging||COPYING_PRESERVES_HFS_DATA + + + $class + + CF$UID + 50 + + NS.string + Packaging||PRIVATE_HEADERS_FOLDER_PATH + + + $class + + CF$UID + 50 + + NS.string + Packaging||PRODUCT_NAME + + + $class + + CF$UID + 50 + + NS.string + Packaging||PLIST_FILE_OUTPUT_FORMAT + + + $class + + CF$UID + 50 + + NS.string + Packaging||PUBLIC_HEADERS_FOLDER_PATH + + + $class + + CF$UID + 50 + + NS.string + Packaging||STRINGS_FILE_OUTPUT_ENCODING + + + $class + + CF$UID + 50 + + NS.string + Packaging||WRAPPER_EXTENSION + + + $class + + CF$UID + 50 + + NS.string + Search Paths||ALWAYS_SEARCH_USER_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||FRAMEWORK_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||HEADER_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||LIBRARY_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||REZ_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Search Paths||EXCLUDED_RECURSIVE_SEARCH_PATH_SUBDIRECTORIES + + + $class + + CF$UID + 50 + + NS.string + Search Paths||INCLUDED_RECURSIVE_SEARCH_PATH_SUBDIRECTORIES + + + $class + + CF$UID + 50 + + NS.string + Search Paths||USER_HEADER_SEARCH_PATHS + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||OTHER_TEST_FLAGS + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||TEST_AFTER_BUILD + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||TEST_HOST + + + $class + + CF$UID + 50 + + NS.string + Unit Testing||TEST_RIG + + + $class + + CF$UID + 50 + + NS.string + Versioning||CURRENT_PROJECT_VERSION + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_FILE + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_EXPORT_DECL + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_PREFIX + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_SUFFIX + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSIONING_SYSTEM + + + $class + + CF$UID + 50 + + NS.string + Versioning||VERSION_INFO_BUILDER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_FAST_OBJC_DISPATCH + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SSE3_EXTENSIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SSE41_EXTENSIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SSE42_EXTENSIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_SUPPLEMENTAL_SSE3_INSTRUCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_STRICT_ALIASING + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_GENERATE_DEBUGGING_SYMBOLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_DYNAMIC_NO_PIC + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_INLINES_ARE_PRIVATE_EXTERN + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_KERNEL_DEVELOPMENT + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||LLVM_LTO + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_REUSE_STRINGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_NO_COMMON_BLOCKS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_ENABLE_OBJC_GC + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_FAST_MATH + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_THREADSAFE_STATICS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_SYMBOLS_PRIVATE_EXTERN + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Code Generation||GCC_UNROLL_LOOPS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_CHAR_IS_UNSIGNED_CHAR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_ASM_KEYWORD + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_C_LANGUAGE_STANDARD + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_INPUT_FILETYPE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_CPP_EXCEPTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_CPP_RTTI + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_LINK_WITH_DYNAMIC_LIBRARIES + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_OBJC_EXCEPTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_TRIGRAPHS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_FLOATING_POINT_LIBRARY_CALLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_USE_INDIRECT_FUNCTION_CALLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_USE_REGISTER_FUNCTION_CALLS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_INCREASE_PRECOMPILED_HEADER_SHARING + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_CW_ASM_SYNTAX + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||OTHER_CFLAGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||OTHER_CPLUSPLUSFLAGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_PRECOMPILE_PREFIX_HEADER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_PREFIX_HEADER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_BUILTIN_FUNCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_ENABLE_PASCAL_STRINGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_FORCE_CPU_SUBTYPE_ALL + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_SHORT_ENUMS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Language||GCC_USE_STANDARD_INCLUDE_SEARCHING + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Preprocessing||GCC_PREPROCESSOR_DEFINITIONS_NOT_USED_IN_PRECOMPS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_CHECK_SWITCH_STATEMENTS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_FOUR_CHARACTER_CONSTANTS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_SHADOW + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_64_TO_32_BIT_CONVERSION + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ALLOW_INCOMPLETE_PROTOCOL + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_INHIBIT_ALL_WARNINGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_INITIALIZER_NOT_FULLY_BRACKETED + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_RETURN_TYPE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_MISSING_PARENTHESES + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_MISSING_FIELD_INITIALIZERS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_MISSING_PROTOTYPES + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_MISSING_NEWLINE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_MULTIPLE_DEFINITION_TYPES_FOR_SELECTOR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_NON_VIRTUAL_DESTRUCTOR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||WARNING_CFLAGS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_HIDDEN_VIRTUAL_FUNCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_PEDANTIC + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_POINTER_SIGNEDNESS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_SIGN_COMPARE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_STRICT_SELECTOR_MATCH + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_TREAT_INCOMPATIBLE_POINTER_TYPE_WARNINGS_AS_ERRORS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_TREAT_IMPLICIT_FUNCTION_DECLARATIONS_AS_ERRORS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_TREAT_WARNINGS_AS_ERRORS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_TYPECHECK_CALLS_TO_PRINTF + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNDECLARED_SELECTOR + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNINITIALIZED_AUTOS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNKNOWN_PRAGMAS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_FUNCTION + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_LABEL + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_PARAMETER + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_VALUE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_UNUSED_VARIABLE + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_DEPRECATED_FUNCTIONS + + + $class + + CF$UID + 50 + + NS.string + LLVM compiler 2.1 - Warnings||GCC_WARN_ABOUT_INVALID_OFFSETOF_MACRO + + + $class + + CF$UID + 95 + + NS.keys + + NS.objects + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 614 + + + CF$UID + 616 + + + CF$UID + 618 + + + CF$UID + 619 + + + CF$UID + 621 + + + CF$UID + 623 + + + CF$UID + 625 + + + NS.objects + + + CF$UID + 627 + + + CF$UID + 635 + + + CF$UID + 638 + + + CF$UID + 642 + + + CF$UID + 645 + + + CF$UID + 649 + + + CF$UID + 652 + + + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 615 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/moses/src/ThreadPool.h + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 617 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/moses/src/ThreadPool.cpp + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 49 + + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 620 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/StatisticsBasedScorer.h + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 622 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/extractor.cpp + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 624 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/mert.cpp + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 626 + + + + $class + + CF$UID + 50 + + NS.string + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/StatisticsBasedScorer.cpp + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 628 + + + CF$UID + 629 + + + CF$UID + 630 + + + CF$UID + 631 + + + NS.objects + + + CF$UID + 632 + + + CF$UID + 633 + + + CF$UID + 16 + + + CF$UID + 634 + + + + PrimaryDocumentTimestamp + PrimaryDocumentVisibleCharacterRange + HideAllIssues + PrimaryDocumentSelectedCharacterRange + 363694733.737234 + {0, 1387} + {0, 0} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 628 + + + CF$UID + 629 + + + CF$UID + 630 + + + CF$UID + 631 + + + NS.objects + + + CF$UID + 636 + + + CF$UID + 637 + + + CF$UID + 16 + + + CF$UID + 634 + + + + 363694729.53642899 + {0, 1485} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 82 + + + CF$UID + 83 + + + CF$UID + 84 + + + CF$UID + 85 + + + NS.objects + + + CF$UID + 639 + + + CF$UID + 640 + + + CF$UID + 16 + + + CF$UID + 641 + + + + 363696391.20240802 + {0, 1309} + {332, 0} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 628 + + + CF$UID + 629 + + + CF$UID + 630 + + + CF$UID + 631 + + + NS.objects + + + CF$UID + 643 + + + CF$UID + 644 + + + CF$UID + 16 + + + CF$UID + 634 + + + + 363694750.12241 + {0, 1049} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 628 + + + CF$UID + 629 + + + CF$UID + 630 + + + CF$UID + 631 + + + NS.objects + + + CF$UID + 646 + + + CF$UID + 647 + + + CF$UID + 16 + + + CF$UID + 648 + + + + 363694727.34139502 + {992, 1572} + {247, 0} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 628 + + + CF$UID + 629 + + + CF$UID + 630 + + + CF$UID + 631 + + + NS.objects + + + CF$UID + 650 + + + CF$UID + 651 + + + CF$UID + 16 + + + CF$UID + 634 + + + + 363627943.92405301 + {0, 1056} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 628 + + + CF$UID + 629 + + + CF$UID + 630 + + + CF$UID + 631 + + + NS.objects + + + CF$UID + 653 + + + CF$UID + 654 + + + CF$UID + 16 + + + CF$UID + 634 + + + + 363694734.10040599 + {0, 1404} + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 656 + + + CF$UID + 657 + + + NS.objects + + + CF$UID + 658 + + + CF$UID + 659 + + + + IDEDeviceLocation + IDEDeviceArchitecture + dvtdevice-local-computer:localhost + x86_64 + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 661 + + + NS.objects + + + CF$UID + 662 + + + + IDENameString + extractor + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 664 + + + CF$UID + 665 + + + CF$UID + 666 + + + NS.objects + + + CF$UID + 667 + + + CF$UID + 693 + + + CF$UID + 426 + + + + IDEActivityReportCompletionSummaryStringSegments + IDEActivityReportOptions + IDEActivityReportTitle + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 668 + + + CF$UID + 675 + + + CF$UID + 679 + + + CF$UID + 684 + + + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 669 + + + CF$UID + 670 + + + CF$UID + 671 + + + NS.objects + + + CF$UID + 672 + + + CF$UID + 673 + + + CF$UID + 674 + + + + IDEActivityReportStringSegmentPriority + IDEActivityReportStringSegmentBackSeparator + IDEActivityReportStringSegmentStringValue + 2 + + Build + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 669 + + + CF$UID + 670 + + + CF$UID + 671 + + + NS.objects + + + CF$UID + 676 + + + CF$UID + 677 + + + CF$UID + 678 + + + + 4 + : + extractor + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 669 + + + CF$UID + 670 + + + CF$UID + 671 + + + NS.objects + + + CF$UID + 680 + + + CF$UID + 681 + + + CF$UID + 682 + + + + 1 + │ + + $class + + CF$UID + 683 + + NS.data + + YnBsaXN0MDDUAQIDBAUGOzxYJHZlcnNpb25YJG9iamVjdHNZJGFy + Y2hpdmVyVCR0b3ASAAGGoK0HCA8QGhscJCUrMTQ3VSRudWxs0wkK + CwwNDlxOU0F0dHJpYnV0ZXNWJGNsYXNzWE5TU3RyaW5ngAOADIAC + WVN1Y2NlZWRlZNMKERITFBdXTlMua2V5c1pOUy5vYmplY3RzgAui + FRaABIAFohgZgAaACVZOU0ZvbnRXTlNDb2xvctQKHR4fICEiI1ZO + U05hbWVWTlNTaXplWE5TZkZsYWdzgAiAByNAJgAAAAAAABENEF8Q + EUx1Y2lkYUdyYW5kZS1Cb2xk0iYnKClaJGNsYXNzbmFtZVgkY2xh + c3Nlc1ZOU0ZvbnSiKCpYTlNPYmplY3TTCiwtLi8wXE5TQ29sb3JT + cGFjZVdOU1doaXRlgAoQA0IwANImJzIzV05TQ29sb3KiMirSJic1 + NlxOU0RpY3Rpb25hcnmiNSrSJic4OV8QEk5TQXR0cmlidXRlZFN0 + cmluZ6I6Kl8QEk5TQXR0cmlidXRlZFN0cmluZ18QD05TS2V5ZWRB + cmNoaXZlctE9PlRyb290gAEACAARABoAIwAtADIANwBFAEsAUgBf + AGYAbwBxAHMAdQB/AIYAjgCZAJsAngCgAKIApQCnAKkAsAC4AMEA + yADPANgA2gDcAOUA6AD8AQEBDAEVARwBHwEoAS8BPAFEAUYBSAFL + AVABWAFbAWABbQFwAXUBigGNAaIBtAG3AbwAAAAAAAACAQAAAAAA + AAA/AAAAAAAAAAAAAAAAAAABvg== + + + + $classes + + NSMutableData + NSData + NSObject + + $classname + NSMutableData + + + $class + + CF$UID + 95 + + NS.keys + + + CF$UID + 669 + + + CF$UID + 685 + + + CF$UID + 686 + + + CF$UID + 671 + + + CF$UID + 687 + + + CF$UID + 688 + + + NS.objects + + + CF$UID + 689 + + + CF$UID + 105 + + + CF$UID + 690 + + + CF$UID + 692 + + + CF$UID + 105 + + + CF$UID + 105 + + + + IDEActivityReportStringSegmentType + IDEActivityReportStringSegmentDate + IDEActivityReportStringSegmentDateStyle + IDEActivityReportStringSegmentTimeStyle + 3 + + $class + + CF$UID + 691 + + NS.time + 363631454.18081301 + + + $classes + + NSDate + NSObject + + $classname + NSDate + + Yesterday at 17:44 + 106 + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 2 + + + + + $class + + CF$UID + 121 + + NS.objects + + + CF$UID + 696 + + + CF$UID + 698 + + + CF$UID + 700 + + + CF$UID + 702 + + + CF$UID + 704 + + + CF$UID + 706 + + + CF$UID + 707 + + + CF$UID + 709 + + + CF$UID + 711 + + + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 697 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/InterpolatedScorer.h + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 699 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/StatisticsBasedScorer.h + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 701 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/StatisticsBasedScorer.cpp + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 703 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/moses/src/ThreadPool.h + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 705 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/moses/src/ThreadPool.cpp + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 423 + + + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 708 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/extractor.cpp + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 710 + + + file://localhost/Users/hieuhoang/unison/workspace/github/hieuhoang/mert/mert.cpp + + $class + + CF$UID + 94 + + NS.base + + CF$UID + 0 + + NS.relative + + CF$UID + 214 + + + + $top + + State + + CF$UID + 1 + + + $version + 100000 + + diff --git a/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcdebugger/Breakpoints.xcbkptlist b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcdebugger/Breakpoints.xcbkptlist new file mode 100644 index 000000000..5029ca7bd --- /dev/null +++ b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcdebugger/Breakpoints.xcbkptlist @@ -0,0 +1,35 @@ + + + + + + + + + diff --git a/contrib/other-builds/moses.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/moses.xcscheme b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/extractor.xcscheme similarity index 62% rename from contrib/other-builds/moses.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/moses.xcscheme rename to contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/extractor.xcscheme index 0d05923ed..48258bc54 100644 --- a/contrib/other-builds/moses.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/moses.xcscheme +++ b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/extractor.xcscheme @@ -13,10 +13,10 @@ buildForAnalyzing = "YES"> + BlueprintIdentifier = "1E1D825E15AC640800FE42E9" + BuildableName = "extractor" + BlueprintName = "extractor" + ReferencedContainer = "container:mert.xcodeproj"> @@ -35,6 +35,15 @@ launchStyle = "0" useCustomWorkingDirectory = "NO" buildConfiguration = "Debug"> + + + + @@ -43,6 +52,15 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" buildConfiguration = "Release"> + + + + diff --git a/contrib/other-builds/OnDiskPt.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/OnDiskPt.xcscheme b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/mert.xcscheme similarity index 63% rename from contrib/other-builds/OnDiskPt.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/OnDiskPt.xcscheme rename to contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/mert.xcscheme index aab0ec3b9..2d41b933c 100644 --- a/contrib/other-builds/OnDiskPt.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/OnDiskPt.xcscheme +++ b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/mert.xcscheme @@ -13,10 +13,10 @@ buildForAnalyzing = "YES"> + BlueprintIdentifier = "1EB0AF041593A2180007E2A4" + BuildableName = "mert" + BlueprintName = "mert" + ReferencedContainer = "container:mert.xcodeproj"> @@ -35,6 +35,15 @@ launchStyle = "0" useCustomWorkingDirectory = "NO" buildConfiguration = "Debug"> + + + + @@ -43,6 +52,15 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" buildConfiguration = "Release"> + + + + diff --git a/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 000000000..d55559c75 --- /dev/null +++ b/contrib/other-builds/mert.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,32 @@ + + + + + SchemeUserState + + extractor.xcscheme + + orderHint + 1 + + mert.xcscheme + + orderHint + 2 + + + SuppressBuildableAutocreation + + 1E1D825E15AC640800FE42E9 + + primary + + + 1EB0AF041593A2180007E2A4 + + primary + + + + + diff --git a/contrib/other-builds/moses-chart-cmd.xcodeproj/project.pbxproj b/contrib/other-builds/moses-chart-cmd.xcodeproj/project.pbxproj index 82fe6607c..775795dee 100644 --- a/contrib/other-builds/moses-chart-cmd.xcodeproj/project.pbxproj +++ b/contrib/other-builds/moses-chart-cmd.xcodeproj/project.pbxproj @@ -308,6 +308,7 @@ ../../irstlm/lib, ../../srilm/lib/macosx, /opt/local/lib, + ../../cmph/lib, ); OTHER_LDFLAGS = ( "-lz", @@ -318,6 +319,9 @@ "-lflm", "-llattice", "-lboost_thread-mt", + "-lboost_filesystem-mt", + "-lboost_system-mt", + "-lcmph", ); PRODUCT_NAME = "moses-chart-cmd"; USER_HEADER_SEARCH_PATHS = "../../ ../../moses/src"; @@ -341,6 +345,7 @@ ../../irstlm/lib, ../../srilm/lib/macosx, /opt/local/lib, + ../../cmph/lib, ); OTHER_LDFLAGS = ( "-lz", @@ -351,6 +356,9 @@ "-lflm", "-llattice", "-lboost_thread-mt", + "-lboost_filesystem-mt", + "-lboost_system-mt", + "-lcmph", ); PRODUCT_NAME = "moses-chart-cmd"; USER_HEADER_SEARCH_PATHS = "../../ ../../moses/src"; diff --git a/contrib/other-builds/moses-cmd.vcxproj b/contrib/other-builds/moses-cmd.vcxproj index 66931907b..3f24ebbdf 100644 --- a/contrib/other-builds/moses-cmd.vcxproj +++ b/contrib/other-builds/moses-cmd.vcxproj @@ -43,6 +43,10 @@ $(SolutionDir)$(Configuration)\ $(Configuration)\ false + C:\Program Files\boost\boost_1_47;$(IncludePath) + C:\Program Files\boost\boost_1_47;$(IncludePath) + C:\Program Files\boost\boost_1_47\lib;$(LibraryPath) + C:\Program Files\boost\boost_1_47\lib;$(LibraryPath) diff --git a/contrib/other-builds/moses-cmd.xcodeproj/project.pbxproj b/contrib/other-builds/moses-cmd.xcodeproj/project.pbxproj index 619ecf76c..aac225ced 100644 --- a/contrib/other-builds/moses-cmd.xcodeproj/project.pbxproj +++ b/contrib/other-builds/moses-cmd.xcodeproj/project.pbxproj @@ -326,15 +326,20 @@ ../../irstlm/lib, ../../srilm/lib/macosx, /opt/local/lib, + ../../cmph/lib, ); OTHER_LDFLAGS = ( - "-lflm", - "-lmisc", - "-loolm", - "-ldstruct", "-lz", "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", "-lboost_thread-mt", + "-lboost_filesystem-mt", + "-lboost_system-mt", + "-lcmph", ); PREBINDING = NO; PRODUCT_NAME = "moses-cmd"; @@ -369,15 +374,20 @@ ../../irstlm/lib, ../../srilm/lib/macosx, /opt/local/lib, + ../../cmph/lib, ); OTHER_LDFLAGS = ( - "-lflm", - "-lmisc", - "-loolm", - "-ldstruct", "-lz", "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", "-lboost_thread-mt", + "-lboost_filesystem-mt", + "-lboost_system-mt", + "-lcmph", ); PREBINDING = NO; PRODUCT_NAME = "moses-cmd"; @@ -409,15 +419,20 @@ ../../irstlm/lib, ../../srilm/lib/macosx, /opt/local/lib, + ../../cmph/lib, ); OTHER_LDFLAGS = ( - "-lflm", - "-lmisc", - "-loolm", - "-ldstruct", "-lz", "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", "-lboost_thread-mt", + "-lboost_filesystem-mt", + "-lboost_system-mt", + "-lcmph", ); PREBINDING = NO; PRODUCT_NAME = "moses-cmd"; diff --git a/contrib/other-builds/moses-cmd.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/moses-cmd.xcscheme b/contrib/other-builds/moses-cmd.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/moses-cmd.xcscheme new file mode 100644 index 000000000..80894ecca --- /dev/null +++ b/contrib/other-builds/moses-cmd.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/moses-cmd.xcscheme @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/other-builds/moses-cmd.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist b/contrib/other-builds/moses-cmd.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 000000000..29af8ddb4 --- /dev/null +++ b/contrib/other-builds/moses-cmd.xcodeproj/xcuserdata/hieuhoang.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,22 @@ + + + + + SchemeUserState + + moses-cmd.xcscheme + + orderHint + 2 + + + SuppressBuildableAutocreation + + 8DD76F620486A84900D96B5E + + primary + + + + + diff --git a/contrib/other-builds/moses-cmd/.cproject b/contrib/other-builds/moses-cmd/.cproject index 53c112cb8..cdad4ad64 100644 --- a/contrib/other-builds/moses-cmd/.cproject +++ b/contrib/other-builds/moses-cmd/.cproject @@ -25,17 +25,27 @@ @@ -51,8 +61,11 @@ + @@ -122,12 +135,13 @@ + - + - + diff --git a/contrib/other-builds/moses.sln b/contrib/other-builds/moses.sln index 37a82495e..a9ea31234 100644 --- a/contrib/other-builds/moses.sln +++ b/contrib/other-builds/moses.sln @@ -20,6 +20,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CreateOnDisk", "CreateOnDis EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "kenlm", "kenlm.vcxproj", "{A5402E0B-6ED7-465C-9669-E4124A0CDDCB}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mosesserver", "mosesserver.vcxproj", "{85811FDF-8AD1-4490-A545-B2F51931A18C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -39,11 +41,17 @@ Global {E2233DB1-5592-46FE-9420-E529420612FA}.Release|Win32.ActiveCfg = Release|Win32 {E2233DB1-5592-46FE-9420-E529420612FA}.Release|Win32.Build.0 = Release|Win32 {88AE90C9-72D2-42ED-8389-770ACDCD4308}.Debug|Win32.ActiveCfg = Debug|Win32 + {88AE90C9-72D2-42ED-8389-770ACDCD4308}.Debug|Win32.Build.0 = Debug|Win32 {88AE90C9-72D2-42ED-8389-770ACDCD4308}.Release|Win32.ActiveCfg = Release|Win32 + {88AE90C9-72D2-42ED-8389-770ACDCD4308}.Release|Win32.Build.0 = Release|Win32 {A5402E0B-6ED7-465C-9669-E4124A0CDDCB}.Debug|Win32.ActiveCfg = Debug|Win32 {A5402E0B-6ED7-465C-9669-E4124A0CDDCB}.Debug|Win32.Build.0 = Debug|Win32 {A5402E0B-6ED7-465C-9669-E4124A0CDDCB}.Release|Win32.ActiveCfg = Release|Win32 {A5402E0B-6ED7-465C-9669-E4124A0CDDCB}.Release|Win32.Build.0 = Release|Win32 + {85811FDF-8AD1-4490-A545-B2F51931A18C}.Debug|Win32.ActiveCfg = Debug|Win32 + {85811FDF-8AD1-4490-A545-B2F51931A18C}.Debug|Win32.Build.0 = Debug|Win32 + {85811FDF-8AD1-4490-A545-B2F51931A18C}.Release|Win32.ActiveCfg = Release|Win32 + {85811FDF-8AD1-4490-A545-B2F51931A18C}.Release|Win32.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/contrib/other-builds/moses.vcxproj b/contrib/other-builds/moses.vcxproj index a7a3d2924..4dba07493 100644 --- a/contrib/other-builds/moses.vcxproj +++ b/contrib/other-builds/moses.vcxproj @@ -13,6 +13,7 @@ + @@ -162,6 +163,7 @@ + @@ -319,13 +321,13 @@ $(Configuration)\ $(SolutionDir)$(Configuration)\ $(Configuration)\ - C:\GnuWin32\include;C:\Program Files\boost\boost_1_47;$(IncludePath) - C:\GnuWin32\include;C:\Program Files\boost\boost_1_47;$(IncludePath) + C:\Program Files\boost\boost_1_47;C:\GnuWin32\include;$(IncludePath) + C:\Program Files\boost\boost_1_47;C:\GnuWin32\include;$(IncludePath) Disabled - $(SolutionDir)\..\..\lm\msinttypes;C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../../;%(AdditionalIncludeDirectories) + C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../../;%(AdditionalIncludeDirectories) WITH_THREADS;NO_PIPES;WIN32;_DEBUG;_CONSOLE;TRACE_ENABLE;_CRT_SECURE_NO_DEPRECATE;_SCL_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) true EnableFastChecks @@ -344,7 +346,7 @@ AnySuitable true Speed - $(SolutionDir)\..\..\lm\msinttypes;C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../../;%(AdditionalIncludeDirectories) + C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../../;%(AdditionalIncludeDirectories) WITH_THREADS;NO_PIPES;WIN32;NDEBUG;_CONSOLE;LM_INTERNAL;TRACE_ENABLE;_CRT_SECURE_NO_DEPRECATE;_SCL_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions) MultiThreadedDLL diff --git a/contrib/other-builds/moses.xcodeproj/project.pbxproj b/contrib/other-builds/moses.xcodeproj/project.pbxproj index 710d2777f..2864615c6 100644 --- a/contrib/other-builds/moses.xcodeproj/project.pbxproj +++ b/contrib/other-builds/moses.xcodeproj/project.pbxproj @@ -7,8 +7,38 @@ objects = { /* Begin PBXBuildFile section */ + 1E0BA41815B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E0BA41615B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.cpp */; }; + 1E0BA41915B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.h */; }; 1E1D824015AC29BB00FE42E9 /* FileHandler.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E1D823E15AC29BB00FE42E9 /* FileHandler.cpp */; }; 1E1D824115AC29BB00FE42E9 /* FileHandler.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E1D823F15AC29BB00FE42E9 /* FileHandler.h */; }; + 1E365EEA16120F4600BA335B /* ChartTranslationOptions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E365EE816120F4600BA335B /* ChartTranslationOptions.cpp */; }; + 1E365EEB16120F4600BA335B /* ChartTranslationOptions.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E365EE916120F4600BA335B /* ChartTranslationOptions.h */; }; + 1E619EA115B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */; }; + 1E619EA215B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */; }; + 1E6D9FD615D027560064D436 /* BlockHashIndex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FBD15D027560064D436 /* BlockHashIndex.cpp */; }; + 1E6D9FD715D027560064D436 /* BlockHashIndex.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FBE15D027560064D436 /* BlockHashIndex.h */; }; + 1E6D9FD815D027560064D436 /* CanonicalHuffman.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FBF15D027560064D436 /* CanonicalHuffman.h */; }; + 1E6D9FD915D027560064D436 /* CmphStringVectorAdapter.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FC015D027560064D436 /* CmphStringVectorAdapter.cpp */; }; + 1E6D9FDA15D027560064D436 /* CmphStringVectorAdapter.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FC115D027560064D436 /* CmphStringVectorAdapter.h */; }; + 1E6D9FDB15D027560064D436 /* ConsistantPhrases.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FC215D027560064D436 /* ConsistantPhrases.h */; }; + 1E6D9FDD15D027560064D436 /* LexicalReorderingTableCompact.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FC415D027560064D436 /* LexicalReorderingTableCompact.cpp */; }; + 1E6D9FDE15D027560064D436 /* LexicalReorderingTableCompact.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FC515D027560064D436 /* LexicalReorderingTableCompact.h */; }; + 1E6D9FDF15D027560064D436 /* LexicalReorderingTableCreator.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FC615D027560064D436 /* LexicalReorderingTableCreator.cpp */; }; + 1E6D9FE015D027560064D436 /* LexicalReorderingTableCreator.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FC715D027560064D436 /* LexicalReorderingTableCreator.h */; }; + 1E6D9FE115D027560064D436 /* ListCoders.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FC815D027560064D436 /* ListCoders.h */; }; + 1E6D9FE215D027560064D436 /* MmapAllocator.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FC915D027560064D436 /* MmapAllocator.h */; }; + 1E6D9FE315D027560064D436 /* MonotonicVector.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FCA15D027560064D436 /* MonotonicVector.h */; }; + 1E6D9FE415D027560064D436 /* MurmurHash3.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FCB15D027560064D436 /* MurmurHash3.cpp */; }; + 1E6D9FE515D027560064D436 /* MurmurHash3.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FCC15D027560064D436 /* MurmurHash3.h */; }; + 1E6D9FE615D027560064D436 /* PackedArray.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FCD15D027560064D436 /* PackedArray.h */; }; + 1E6D9FE715D027560064D436 /* PhraseDecoder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FCE15D027560064D436 /* PhraseDecoder.cpp */; }; + 1E6D9FE815D027560064D436 /* PhraseDecoder.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FCF15D027560064D436 /* PhraseDecoder.h */; }; + 1E6D9FE915D027560064D436 /* PhraseDictionaryCompact.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FD015D027560064D436 /* PhraseDictionaryCompact.cpp */; }; + 1E6D9FEA15D027560064D436 /* PhraseDictionaryCompact.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FD115D027560064D436 /* PhraseDictionaryCompact.h */; }; + 1E6D9FEB15D027560064D436 /* PhraseTableCreator.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E6D9FD215D027560064D436 /* PhraseTableCreator.cpp */; }; + 1E6D9FEC15D027560064D436 /* PhraseTableCreator.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FD315D027560064D436 /* PhraseTableCreator.h */; }; + 1E6D9FED15D027560064D436 /* StringVector.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FD415D027560064D436 /* StringVector.h */; }; + 1E6D9FEE15D027560064D436 /* TargetPhraseCollectionCache.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E6D9FD515D027560064D436 /* TargetPhraseCollectionCache.h */; }; 1E879EA715A346F90051F346 /* SearchNormalBatch.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */; }; 1E879EA815A346F90051F346 /* SearchNormalBatch.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E879EA615A346F90051F346 /* SearchNormalBatch.h */; }; 1EAC363514CDC79300DF97C3 /* Loader.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EAC362C14CDC79300DF97C3 /* Loader.h */; }; @@ -20,6 +50,8 @@ 1EAC363B14CDC79300DF97C3 /* LoaderHiero.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EAC363214CDC79300DF97C3 /* LoaderHiero.h */; }; 1EAC363C14CDC79300DF97C3 /* LoaderStandard.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EAC363314CDC79300DF97C3 /* LoaderStandard.cpp */; }; 1EAC363D14CDC79300DF97C3 /* LoaderStandard.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EAC363414CDC79300DF97C3 /* LoaderStandard.h */; }; + 1EC32DB815D2D90700A313B1 /* ThrowingFwrite.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC32DB615D2D90700A313B1 /* ThrowingFwrite.cpp */; }; + 1EC32DB915D2D90700A313B1 /* ThrowingFwrite.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC32DB715D2D90700A313B1 /* ThrowingFwrite.h */; }; 1EC7374614B977AB00238410 /* AlignmentInfo.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735D314B977AA00238410 /* AlignmentInfo.cpp */; }; 1EC7374714B977AB00238410 /* AlignmentInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735D414B977AA00238410 /* AlignmentInfo.h */; }; 1EC7374814B977AB00238410 /* AlignmentInfoCollection.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735D514B977AA00238410 /* AlignmentInfoCollection.cpp */; }; @@ -28,7 +60,6 @@ 1EC7374B14B977AB00238410 /* BilingualDynSuffixArray.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735D814B977AA00238410 /* BilingualDynSuffixArray.h */; }; 1EC7374C14B977AB00238410 /* BitmapContainer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735D914B977AA00238410 /* BitmapContainer.cpp */; }; 1EC7374D14B977AB00238410 /* BitmapContainer.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735DA14B977AA00238410 /* BitmapContainer.h */; }; - 1EC7374E14B977AB00238410 /* CellCollection.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735DB14B977AA00238410 /* CellCollection.h */; }; 1EC7374F14B977AB00238410 /* ChartCell.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735DC14B977AA00238410 /* ChartCell.cpp */; }; 1EC7375014B977AB00238410 /* ChartCell.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735DD14B977AA00238410 /* ChartCell.h */; }; 1EC7375114B977AB00238410 /* ChartCellCollection.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735DE14B977AA00238410 /* ChartCellCollection.cpp */; }; @@ -42,10 +73,6 @@ 1EC7375914B977AB00238410 /* ChartManager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735E614B977AA00238410 /* ChartManager.cpp */; }; 1EC7375A14B977AB00238410 /* ChartManager.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735E714B977AA00238410 /* ChartManager.h */; }; 1EC7375C14B977AB00238410 /* ChartRuleLookupManager.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735E914B977AA00238410 /* ChartRuleLookupManager.h */; }; - 1EC7376114B977AB00238410 /* ChartTranslationOption.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735EE14B977AA00238410 /* ChartTranslationOption.cpp */; }; - 1EC7376214B977AB00238410 /* ChartTranslationOption.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735EF14B977AA00238410 /* ChartTranslationOption.h */; }; - 1EC7376314B977AB00238410 /* ChartTranslationOptionCollection.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735F014B977AA00238410 /* ChartTranslationOptionCollection.cpp */; }; - 1EC7376414B977AB00238410 /* ChartTranslationOptionCollection.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735F114B977AA00238410 /* ChartTranslationOptionCollection.h */; }; 1EC7376514B977AB00238410 /* ChartTranslationOptionList.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735F214B977AA00238410 /* ChartTranslationOptionList.cpp */; }; 1EC7376614B977AB00238410 /* ChartTranslationOptionList.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EC735F314B977AA00238410 /* ChartTranslationOptionList.h */; }; 1EC7376714B977AB00238410 /* ChartTrellisDetour.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EC735F414B977AA00238410 /* ChartTrellisDetour.cpp */; }; @@ -295,14 +322,53 @@ 1EDA809114D19FBF003D2191 /* UTrie.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EDA808314D19FBF003D2191 /* UTrie.h */; }; 1EDA809214D19FBF003D2191 /* UTrieNode.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EDA808414D19FBF003D2191 /* UTrieNode.cpp */; }; 1EDA809314D19FBF003D2191 /* UTrieNode.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EDA808514D19FBF003D2191 /* UTrieNode.h */; }; + 1EE418ED15C7FDCB0028F9AB /* Match.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EE418E415C7FDCB0028F9AB /* Match.h */; }; + 1EE418EE15C7FDCB0028F9AB /* SentenceAlignment.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EE418E515C7FDCB0028F9AB /* SentenceAlignment.cpp */; }; + 1EE418EF15C7FDCB0028F9AB /* SentenceAlignment.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EE418E615C7FDCB0028F9AB /* SentenceAlignment.h */; }; + 1EE418F015C7FDCB0028F9AB /* SuffixArray.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EE418E715C7FDCB0028F9AB /* SuffixArray.cpp */; }; + 1EE418F115C7FDCB0028F9AB /* SuffixArray.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EE418E815C7FDCB0028F9AB /* SuffixArray.h */; }; + 1EE418F215C7FDCB0028F9AB /* FuzzyMatchWrapper.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EE418E915C7FDCB0028F9AB /* FuzzyMatchWrapper.cpp */; }; + 1EE418F315C7FDCB0028F9AB /* FuzzyMatchWrapper.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EE418EA15C7FDCB0028F9AB /* FuzzyMatchWrapper.h */; }; + 1EE418F415C7FDCB0028F9AB /* Vocabulary.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EE418EB15C7FDCB0028F9AB /* Vocabulary.cpp */; }; + 1EE418F515C7FDCB0028F9AB /* Vocabulary.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EE418EC15C7FDCB0028F9AB /* Vocabulary.h */; }; 1EF0709314B9EFCC0052152A /* ParallelBackoff.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EF0709114B9EFCC0052152A /* ParallelBackoff.cpp */; }; 1EF0709414B9EFCC0052152A /* ParallelBackoff.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EF0709214B9EFCC0052152A /* ParallelBackoff.h */; }; 1EF8F2C4159A61970047B613 /* HypoList.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EF8F2C3159A61970047B613 /* HypoList.h */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 1E0BA41615B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionaryFuzzyMatch.cpp; path = ../../moses/src/RuleTable/PhraseDictionaryFuzzyMatch.cpp; sourceTree = ""; }; + 1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryFuzzyMatch.h; path = ../../moses/src/RuleTable/PhraseDictionaryFuzzyMatch.h; sourceTree = ""; }; 1E1D823E15AC29BB00FE42E9 /* FileHandler.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FileHandler.cpp; sourceTree = ""; }; 1E1D823F15AC29BB00FE42E9 /* FileHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FileHandler.h; sourceTree = ""; }; + 1E365EE816120F4600BA335B /* ChartTranslationOptions.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartTranslationOptions.cpp; path = ../../moses/src/ChartTranslationOptions.cpp; sourceTree = ""; }; + 1E365EE916120F4600BA335B /* ChartTranslationOptions.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartTranslationOptions.h; path = ../../moses/src/ChartTranslationOptions.h; sourceTree = ""; }; + 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartRuleLookupManagerMemoryPerSentence.cpp; path = ../../moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp; sourceTree = ""; }; + 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartRuleLookupManagerMemoryPerSentence.h; path = ../../moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h; sourceTree = ""; }; + 1E6D9FBD15D027560064D436 /* BlockHashIndex.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = BlockHashIndex.cpp; path = ../../moses/src/CompactPT/BlockHashIndex.cpp; sourceTree = ""; }; + 1E6D9FBE15D027560064D436 /* BlockHashIndex.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = BlockHashIndex.h; path = ../../moses/src/CompactPT/BlockHashIndex.h; sourceTree = ""; }; + 1E6D9FBF15D027560064D436 /* CanonicalHuffman.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = CanonicalHuffman.h; path = ../../moses/src/CompactPT/CanonicalHuffman.h; sourceTree = ""; }; + 1E6D9FC015D027560064D436 /* CmphStringVectorAdapter.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = CmphStringVectorAdapter.cpp; path = ../../moses/src/CompactPT/CmphStringVectorAdapter.cpp; sourceTree = ""; }; + 1E6D9FC115D027560064D436 /* CmphStringVectorAdapter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = CmphStringVectorAdapter.h; path = ../../moses/src/CompactPT/CmphStringVectorAdapter.h; sourceTree = ""; }; + 1E6D9FC215D027560064D436 /* ConsistantPhrases.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ConsistantPhrases.h; path = ../../moses/src/CompactPT/ConsistantPhrases.h; sourceTree = ""; }; + 1E6D9FC415D027560064D436 /* LexicalReorderingTableCompact.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexicalReorderingTableCompact.cpp; path = ../../moses/src/CompactPT/LexicalReorderingTableCompact.cpp; sourceTree = ""; }; + 1E6D9FC515D027560064D436 /* LexicalReorderingTableCompact.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LexicalReorderingTableCompact.h; path = ../../moses/src/CompactPT/LexicalReorderingTableCompact.h; sourceTree = ""; }; + 1E6D9FC615D027560064D436 /* LexicalReorderingTableCreator.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexicalReorderingTableCreator.cpp; path = ../../moses/src/CompactPT/LexicalReorderingTableCreator.cpp; sourceTree = ""; }; + 1E6D9FC715D027560064D436 /* LexicalReorderingTableCreator.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LexicalReorderingTableCreator.h; path = ../../moses/src/CompactPT/LexicalReorderingTableCreator.h; sourceTree = ""; }; + 1E6D9FC815D027560064D436 /* ListCoders.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ListCoders.h; path = ../../moses/src/CompactPT/ListCoders.h; sourceTree = ""; }; + 1E6D9FC915D027560064D436 /* MmapAllocator.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = MmapAllocator.h; path = ../../moses/src/CompactPT/MmapAllocator.h; sourceTree = ""; }; + 1E6D9FCA15D027560064D436 /* MonotonicVector.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = MonotonicVector.h; path = ../../moses/src/CompactPT/MonotonicVector.h; sourceTree = ""; }; + 1E6D9FCB15D027560064D436 /* MurmurHash3.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = MurmurHash3.cpp; path = ../../moses/src/CompactPT/MurmurHash3.cpp; sourceTree = ""; }; + 1E6D9FCC15D027560064D436 /* MurmurHash3.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = MurmurHash3.h; path = ../../moses/src/CompactPT/MurmurHash3.h; sourceTree = ""; }; + 1E6D9FCD15D027560064D436 /* PackedArray.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PackedArray.h; path = ../../moses/src/CompactPT/PackedArray.h; sourceTree = ""; }; + 1E6D9FCE15D027560064D436 /* PhraseDecoder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDecoder.cpp; path = ../../moses/src/CompactPT/PhraseDecoder.cpp; sourceTree = ""; }; + 1E6D9FCF15D027560064D436 /* PhraseDecoder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDecoder.h; path = ../../moses/src/CompactPT/PhraseDecoder.h; sourceTree = ""; }; + 1E6D9FD015D027560064D436 /* PhraseDictionaryCompact.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseDictionaryCompact.cpp; path = ../../moses/src/CompactPT/PhraseDictionaryCompact.cpp; sourceTree = ""; }; + 1E6D9FD115D027560064D436 /* PhraseDictionaryCompact.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryCompact.h; path = ../../moses/src/CompactPT/PhraseDictionaryCompact.h; sourceTree = ""; }; + 1E6D9FD215D027560064D436 /* PhraseTableCreator.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = PhraseTableCreator.cpp; path = ../../moses/src/CompactPT/PhraseTableCreator.cpp; sourceTree = ""; }; + 1E6D9FD315D027560064D436 /* PhraseTableCreator.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseTableCreator.h; path = ../../moses/src/CompactPT/PhraseTableCreator.h; sourceTree = ""; }; + 1E6D9FD415D027560064D436 /* StringVector.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = StringVector.h; path = ../../moses/src/CompactPT/StringVector.h; sourceTree = ""; }; + 1E6D9FD515D027560064D436 /* TargetPhraseCollectionCache.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = TargetPhraseCollectionCache.h; path = ../../moses/src/CompactPT/TargetPhraseCollectionCache.h; sourceTree = ""; }; 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = SearchNormalBatch.cpp; path = ../../moses/src/SearchNormalBatch.cpp; sourceTree = ""; }; 1E879EA615A346F90051F346 /* SearchNormalBatch.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SearchNormalBatch.h; path = ../../moses/src/SearchNormalBatch.h; sourceTree = ""; }; 1EAC362C14CDC79300DF97C3 /* Loader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Loader.h; path = ../../moses/src/RuleTable/Loader.h; sourceTree = ""; }; @@ -314,6 +380,8 @@ 1EAC363214CDC79300DF97C3 /* LoaderHiero.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LoaderHiero.h; path = ../../moses/src/RuleTable/LoaderHiero.h; sourceTree = ""; }; 1EAC363314CDC79300DF97C3 /* LoaderStandard.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LoaderStandard.cpp; path = ../../moses/src/RuleTable/LoaderStandard.cpp; sourceTree = ""; }; 1EAC363414CDC79300DF97C3 /* LoaderStandard.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LoaderStandard.h; path = ../../moses/src/RuleTable/LoaderStandard.h; sourceTree = ""; }; + 1EC32DB615D2D90700A313B1 /* ThrowingFwrite.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ThrowingFwrite.cpp; path = ../../moses/src/CompactPT/ThrowingFwrite.cpp; sourceTree = ""; }; + 1EC32DB715D2D90700A313B1 /* ThrowingFwrite.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ThrowingFwrite.h; path = ../../moses/src/CompactPT/ThrowingFwrite.h; sourceTree = ""; }; 1EC735D314B977AA00238410 /* AlignmentInfo.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = AlignmentInfo.cpp; path = ../../moses/src/AlignmentInfo.cpp; sourceTree = ""; }; 1EC735D414B977AA00238410 /* AlignmentInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AlignmentInfo.h; path = ../../moses/src/AlignmentInfo.h; sourceTree = ""; }; 1EC735D514B977AA00238410 /* AlignmentInfoCollection.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = AlignmentInfoCollection.cpp; path = ../../moses/src/AlignmentInfoCollection.cpp; sourceTree = ""; }; @@ -322,7 +390,6 @@ 1EC735D814B977AA00238410 /* BilingualDynSuffixArray.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = BilingualDynSuffixArray.h; path = ../../moses/src/BilingualDynSuffixArray.h; sourceTree = ""; }; 1EC735D914B977AA00238410 /* BitmapContainer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = BitmapContainer.cpp; path = ../../moses/src/BitmapContainer.cpp; sourceTree = ""; }; 1EC735DA14B977AA00238410 /* BitmapContainer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = BitmapContainer.h; path = ../../moses/src/BitmapContainer.h; sourceTree = ""; }; - 1EC735DB14B977AA00238410 /* CellCollection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = CellCollection.h; path = ../../moses/src/CellCollection.h; sourceTree = ""; }; 1EC735DC14B977AA00238410 /* ChartCell.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartCell.cpp; path = ../../moses/src/ChartCell.cpp; sourceTree = ""; }; 1EC735DD14B977AA00238410 /* ChartCell.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartCell.h; path = ../../moses/src/ChartCell.h; sourceTree = ""; }; 1EC735DE14B977AA00238410 /* ChartCellCollection.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartCellCollection.cpp; path = ../../moses/src/ChartCellCollection.cpp; sourceTree = ""; }; @@ -336,10 +403,6 @@ 1EC735E614B977AA00238410 /* ChartManager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartManager.cpp; path = ../../moses/src/ChartManager.cpp; sourceTree = ""; }; 1EC735E714B977AA00238410 /* ChartManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartManager.h; path = ../../moses/src/ChartManager.h; sourceTree = ""; }; 1EC735E914B977AA00238410 /* ChartRuleLookupManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartRuleLookupManager.h; path = ../../moses/src/ChartRuleLookupManager.h; sourceTree = ""; }; - 1EC735EE14B977AA00238410 /* ChartTranslationOption.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartTranslationOption.cpp; path = ../../moses/src/ChartTranslationOption.cpp; sourceTree = ""; }; - 1EC735EF14B977AA00238410 /* ChartTranslationOption.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartTranslationOption.h; path = ../../moses/src/ChartTranslationOption.h; sourceTree = ""; }; - 1EC735F014B977AA00238410 /* ChartTranslationOptionCollection.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartTranslationOptionCollection.cpp; path = ../../moses/src/ChartTranslationOptionCollection.cpp; sourceTree = ""; }; - 1EC735F114B977AA00238410 /* ChartTranslationOptionCollection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartTranslationOptionCollection.h; path = ../../moses/src/ChartTranslationOptionCollection.h; sourceTree = ""; }; 1EC735F214B977AA00238410 /* ChartTranslationOptionList.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartTranslationOptionList.cpp; path = ../../moses/src/ChartTranslationOptionList.cpp; sourceTree = ""; }; 1EC735F314B977AA00238410 /* ChartTranslationOptionList.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartTranslationOptionList.h; path = ../../moses/src/ChartTranslationOptionList.h; sourceTree = ""; }; 1EC735F414B977AA00238410 /* ChartTrellisDetour.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartTrellisDetour.cpp; path = ../../moses/src/ChartTrellisDetour.cpp; sourceTree = ""; }; @@ -591,6 +654,15 @@ 1EDA808314D19FBF003D2191 /* UTrie.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = UTrie.h; path = ../../moses/src/RuleTable/UTrie.h; sourceTree = ""; }; 1EDA808414D19FBF003D2191 /* UTrieNode.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = UTrieNode.cpp; path = ../../moses/src/RuleTable/UTrieNode.cpp; sourceTree = ""; }; 1EDA808514D19FBF003D2191 /* UTrieNode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = UTrieNode.h; path = ../../moses/src/RuleTable/UTrieNode.h; sourceTree = ""; }; + 1EE418E415C7FDCB0028F9AB /* Match.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Match.h; path = "../../moses/src/fuzzy-match/Match.h"; sourceTree = ""; }; + 1EE418E515C7FDCB0028F9AB /* SentenceAlignment.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = SentenceAlignment.cpp; path = "../../moses/src/fuzzy-match/SentenceAlignment.cpp"; sourceTree = ""; }; + 1EE418E615C7FDCB0028F9AB /* SentenceAlignment.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SentenceAlignment.h; path = "../../moses/src/fuzzy-match/SentenceAlignment.h"; sourceTree = ""; }; + 1EE418E715C7FDCB0028F9AB /* SuffixArray.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = SuffixArray.cpp; path = "../../moses/src/fuzzy-match/SuffixArray.cpp"; sourceTree = ""; }; + 1EE418E815C7FDCB0028F9AB /* SuffixArray.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SuffixArray.h; path = "../../moses/src/fuzzy-match/SuffixArray.h"; sourceTree = ""; }; + 1EE418E915C7FDCB0028F9AB /* FuzzyMatchWrapper.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = FuzzyMatchWrapper.cpp; path = "../../moses/src/fuzzy-match/FuzzyMatchWrapper.cpp"; sourceTree = ""; }; + 1EE418EA15C7FDCB0028F9AB /* FuzzyMatchWrapper.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = FuzzyMatchWrapper.h; path = "../../moses/src/fuzzy-match/FuzzyMatchWrapper.h"; sourceTree = ""; }; + 1EE418EB15C7FDCB0028F9AB /* Vocabulary.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Vocabulary.cpp; path = "../../moses/src/fuzzy-match/Vocabulary.cpp"; sourceTree = ""; }; + 1EE418EC15C7FDCB0028F9AB /* Vocabulary.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Vocabulary.h; path = "../../moses/src/fuzzy-match/Vocabulary.h"; sourceTree = ""; }; 1EF0709114B9EFCC0052152A /* ParallelBackoff.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParallelBackoff.cpp; sourceTree = ""; }; 1EF0709214B9EFCC0052152A /* ParallelBackoff.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ParallelBackoff.h; sourceTree = ""; }; 1EF8F2C3159A61970047B613 /* HypoList.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = HypoList.h; path = ../../moses/src/HypoList.h; sourceTree = ""; }; @@ -621,8 +693,8 @@ 08FB7795FE84155DC02AAC07 /* Source */ = { isa = PBXGroup; children = ( - 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */, - 1E879EA615A346F90051F346 /* SearchNormalBatch.h */, + 1E6D9FF015D027680064D436 /* CompactPT */, + 1ECF13DE15C1A82400EA1DCE /* fuzzy-match */, 1EDA803514D19ECD003D2191 /* Scope3Parser */, 1EDA803414D19EB8003D2191 /* CYKPlusParser */, 1EC7365B14B977AA00238410 /* LM */, @@ -636,7 +708,6 @@ 1EC735D814B977AA00238410 /* BilingualDynSuffixArray.h */, 1EC735D914B977AA00238410 /* BitmapContainer.cpp */, 1EC735DA14B977AA00238410 /* BitmapContainer.h */, - 1EC735DB14B977AA00238410 /* CellCollection.h */, 1EC735DC14B977AA00238410 /* ChartCell.cpp */, 1EC735DD14B977AA00238410 /* ChartCell.h */, 1EC735DE14B977AA00238410 /* ChartCellCollection.cpp */, @@ -650,10 +721,8 @@ 1EC735E614B977AA00238410 /* ChartManager.cpp */, 1EC735E714B977AA00238410 /* ChartManager.h */, 1EC735E914B977AA00238410 /* ChartRuleLookupManager.h */, - 1EC735EE14B977AA00238410 /* ChartTranslationOption.cpp */, - 1EC735EF14B977AA00238410 /* ChartTranslationOption.h */, - 1EC735F014B977AA00238410 /* ChartTranslationOptionCollection.cpp */, - 1EC735F114B977AA00238410 /* ChartTranslationOptionCollection.h */, + 1E365EE816120F4600BA335B /* ChartTranslationOptions.cpp */, + 1E365EE916120F4600BA335B /* ChartTranslationOptions.h */, 1EC735F214B977AA00238410 /* ChartTranslationOptionList.cpp */, 1EC735F314B977AA00238410 /* ChartTranslationOptionList.h */, 1EC735F414B977AA00238410 /* ChartTrellisDetour.cpp */, @@ -782,6 +851,8 @@ 1EC736F414B977AB00238410 /* SearchCubePruning.h */, 1EC736F514B977AB00238410 /* SearchNormal.cpp */, 1EC736F614B977AB00238410 /* SearchNormal.h */, + 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */, + 1E879EA615A346F90051F346 /* SearchNormalBatch.h */, 1EC736F714B977AB00238410 /* Sentence.cpp */, 1EC736F814B977AB00238410 /* Sentence.h */, 1EC736F914B977AB00238410 /* SentenceStats.cpp */, @@ -845,6 +916,39 @@ name = Products; sourceTree = ""; }; + 1E6D9FF015D027680064D436 /* CompactPT */ = { + isa = PBXGroup; + children = ( + 1EC32DB615D2D90700A313B1 /* ThrowingFwrite.cpp */, + 1EC32DB715D2D90700A313B1 /* ThrowingFwrite.h */, + 1E6D9FBD15D027560064D436 /* BlockHashIndex.cpp */, + 1E6D9FBE15D027560064D436 /* BlockHashIndex.h */, + 1E6D9FBF15D027560064D436 /* CanonicalHuffman.h */, + 1E6D9FC015D027560064D436 /* CmphStringVectorAdapter.cpp */, + 1E6D9FC115D027560064D436 /* CmphStringVectorAdapter.h */, + 1E6D9FC215D027560064D436 /* ConsistantPhrases.h */, + 1E6D9FC415D027560064D436 /* LexicalReorderingTableCompact.cpp */, + 1E6D9FC515D027560064D436 /* LexicalReorderingTableCompact.h */, + 1E6D9FC615D027560064D436 /* LexicalReorderingTableCreator.cpp */, + 1E6D9FC715D027560064D436 /* LexicalReorderingTableCreator.h */, + 1E6D9FC815D027560064D436 /* ListCoders.h */, + 1E6D9FC915D027560064D436 /* MmapAllocator.h */, + 1E6D9FCA15D027560064D436 /* MonotonicVector.h */, + 1E6D9FCB15D027560064D436 /* MurmurHash3.cpp */, + 1E6D9FCC15D027560064D436 /* MurmurHash3.h */, + 1E6D9FCD15D027560064D436 /* PackedArray.h */, + 1E6D9FCE15D027560064D436 /* PhraseDecoder.cpp */, + 1E6D9FCF15D027560064D436 /* PhraseDecoder.h */, + 1E6D9FD015D027560064D436 /* PhraseDictionaryCompact.cpp */, + 1E6D9FD115D027560064D436 /* PhraseDictionaryCompact.h */, + 1E6D9FD215D027560064D436 /* PhraseTableCreator.cpp */, + 1E6D9FD315D027560064D436 /* PhraseTableCreator.h */, + 1E6D9FD415D027560064D436 /* StringVector.h */, + 1E6D9FD515D027560064D436 /* TargetPhraseCollectionCache.h */, + ); + name = CompactPT; + sourceTree = ""; + }; 1EAC362B14CDC76200DF97C3 /* RuleTable */ = { isa = PBXGroup; children = ( @@ -856,6 +960,8 @@ 1EDA807D14D19FBF003D2191 /* PhraseDictionaryOnDisk.h */, 1EDA807E14D19FBF003D2191 /* PhraseDictionarySCFG.cpp */, 1EDA807F14D19FBF003D2191 /* PhraseDictionarySCFG.h */, + 1E0BA41615B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.cpp */, + 1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.h */, 1EDA808014D19FBF003D2191 /* Trie.cpp */, 1EDA808114D19FBF003D2191 /* Trie.h */, 1EDA808214D19FBF003D2191 /* UTrie.cpp */, @@ -930,9 +1036,27 @@ path = ../../moses/src/LM; sourceTree = ""; }; + 1ECF13DE15C1A82400EA1DCE /* fuzzy-match */ = { + isa = PBXGroup; + children = ( + 1EE418E415C7FDCB0028F9AB /* Match.h */, + 1EE418E515C7FDCB0028F9AB /* SentenceAlignment.cpp */, + 1EE418E615C7FDCB0028F9AB /* SentenceAlignment.h */, + 1EE418E715C7FDCB0028F9AB /* SuffixArray.cpp */, + 1EE418E815C7FDCB0028F9AB /* SuffixArray.h */, + 1EE418E915C7FDCB0028F9AB /* FuzzyMatchWrapper.cpp */, + 1EE418EA15C7FDCB0028F9AB /* FuzzyMatchWrapper.h */, + 1EE418EB15C7FDCB0028F9AB /* Vocabulary.cpp */, + 1EE418EC15C7FDCB0028F9AB /* Vocabulary.h */, + ); + name = "fuzzy-match"; + sourceTree = ""; + }; 1EDA803414D19EB8003D2191 /* CYKPlusParser */ = { isa = PBXGroup; children = ( + 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */, + 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */, 1EDA806214D19F12003D2191 /* ChartRuleLookupManagerCYKPlus.cpp */, 1EDA806314D19F12003D2191 /* ChartRuleLookupManagerCYKPlus.h */, 1EDA806414D19F12003D2191 /* ChartRuleLookupManagerMemory.cpp */, @@ -986,7 +1110,6 @@ 1EC7374914B977AB00238410 /* AlignmentInfoCollection.h in Headers */, 1EC7374B14B977AB00238410 /* BilingualDynSuffixArray.h in Headers */, 1EC7374D14B977AB00238410 /* BitmapContainer.h in Headers */, - 1EC7374E14B977AB00238410 /* CellCollection.h in Headers */, 1EC7375014B977AB00238410 /* ChartCell.h in Headers */, 1EC7375214B977AB00238410 /* ChartCellCollection.h in Headers */, 1EC7375314B977AB00238410 /* ChartCellLabel.h in Headers */, @@ -995,8 +1118,6 @@ 1EC7375814B977AB00238410 /* ChartHypothesisCollection.h in Headers */, 1EC7375A14B977AB00238410 /* ChartManager.h in Headers */, 1EC7375C14B977AB00238410 /* ChartRuleLookupManager.h in Headers */, - 1EC7376214B977AB00238410 /* ChartTranslationOption.h in Headers */, - 1EC7376414B977AB00238410 /* ChartTranslationOptionCollection.h in Headers */, 1EC7376614B977AB00238410 /* ChartTranslationOptionList.h in Headers */, 1EC7376814B977AB00238410 /* ChartTrellisDetour.h in Headers */, 1EC7376A14B977AB00238410 /* ChartTrellisDetourQueue.h in Headers */, @@ -1143,6 +1264,31 @@ 1EF8F2C4159A61970047B613 /* HypoList.h in Headers */, 1E879EA815A346F90051F346 /* SearchNormalBatch.h in Headers */, 1E1D824115AC29BB00FE42E9 /* FileHandler.h in Headers */, + 1E0BA41915B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.h in Headers */, + 1E619EA215B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h in Headers */, + 1EE418ED15C7FDCB0028F9AB /* Match.h in Headers */, + 1EE418EF15C7FDCB0028F9AB /* SentenceAlignment.h in Headers */, + 1EE418F115C7FDCB0028F9AB /* SuffixArray.h in Headers */, + 1EE418F315C7FDCB0028F9AB /* FuzzyMatchWrapper.h in Headers */, + 1EE418F515C7FDCB0028F9AB /* Vocabulary.h in Headers */, + 1E6D9FD715D027560064D436 /* BlockHashIndex.h in Headers */, + 1E6D9FD815D027560064D436 /* CanonicalHuffman.h in Headers */, + 1E6D9FDA15D027560064D436 /* CmphStringVectorAdapter.h in Headers */, + 1E6D9FDB15D027560064D436 /* ConsistantPhrases.h in Headers */, + 1E6D9FDE15D027560064D436 /* LexicalReorderingTableCompact.h in Headers */, + 1E6D9FE015D027560064D436 /* LexicalReorderingTableCreator.h in Headers */, + 1E6D9FE115D027560064D436 /* ListCoders.h in Headers */, + 1E6D9FE215D027560064D436 /* MmapAllocator.h in Headers */, + 1E6D9FE315D027560064D436 /* MonotonicVector.h in Headers */, + 1E6D9FE515D027560064D436 /* MurmurHash3.h in Headers */, + 1E6D9FE615D027560064D436 /* PackedArray.h in Headers */, + 1E6D9FE815D027560064D436 /* PhraseDecoder.h in Headers */, + 1E6D9FEA15D027560064D436 /* PhraseDictionaryCompact.h in Headers */, + 1E6D9FEC15D027560064D436 /* PhraseTableCreator.h in Headers */, + 1E6D9FED15D027560064D436 /* StringVector.h in Headers */, + 1E6D9FEE15D027560064D436 /* TargetPhraseCollectionCache.h in Headers */, + 1EC32DB915D2D90700A313B1 /* ThrowingFwrite.h in Headers */, + 1E365EEB16120F4600BA335B /* ChartTranslationOptions.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1172,7 +1318,7 @@ 08FB7793FE84155DC02AAC07 /* Project object */ = { isa = PBXProject; attributes = { - LastUpgradeCheck = 0410; + LastUpgradeCheck = 0420; }; buildConfigurationList = 1DEB91EF08733DB70010E9CD /* Build configuration list for PBXProject "moses" */; compatibilityVersion = "Xcode 3.2"; @@ -1207,8 +1353,6 @@ 1EC7375514B977AB00238410 /* ChartHypothesis.cpp in Sources */, 1EC7375714B977AB00238410 /* ChartHypothesisCollection.cpp in Sources */, 1EC7375914B977AB00238410 /* ChartManager.cpp in Sources */, - 1EC7376114B977AB00238410 /* ChartTranslationOption.cpp in Sources */, - 1EC7376314B977AB00238410 /* ChartTranslationOptionCollection.cpp in Sources */, 1EC7376514B977AB00238410 /* ChartTranslationOptionList.cpp in Sources */, 1EC7376714B977AB00238410 /* ChartTrellisDetour.cpp in Sources */, 1EC7376914B977AB00238410 /* ChartTrellisDetourQueue.cpp in Sources */, @@ -1328,6 +1472,22 @@ 1EDA809214D19FBF003D2191 /* UTrieNode.cpp in Sources */, 1E879EA715A346F90051F346 /* SearchNormalBatch.cpp in Sources */, 1E1D824015AC29BB00FE42E9 /* FileHandler.cpp in Sources */, + 1E0BA41815B70E5F00AC70E1 /* PhraseDictionaryFuzzyMatch.cpp in Sources */, + 1E619EA115B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp in Sources */, + 1EE418EE15C7FDCB0028F9AB /* SentenceAlignment.cpp in Sources */, + 1EE418F015C7FDCB0028F9AB /* SuffixArray.cpp in Sources */, + 1EE418F215C7FDCB0028F9AB /* FuzzyMatchWrapper.cpp in Sources */, + 1EE418F415C7FDCB0028F9AB /* Vocabulary.cpp in Sources */, + 1E6D9FD615D027560064D436 /* BlockHashIndex.cpp in Sources */, + 1E6D9FD915D027560064D436 /* CmphStringVectorAdapter.cpp in Sources */, + 1E6D9FDD15D027560064D436 /* LexicalReorderingTableCompact.cpp in Sources */, + 1E6D9FDF15D027560064D436 /* LexicalReorderingTableCreator.cpp in Sources */, + 1E6D9FE415D027560064D436 /* MurmurHash3.cpp in Sources */, + 1E6D9FE715D027560064D436 /* PhraseDecoder.cpp in Sources */, + 1E6D9FE915D027560064D436 /* PhraseDictionaryCompact.cpp in Sources */, + 1E6D9FEB15D027560064D436 /* PhraseTableCreator.cpp in Sources */, + 1EC32DB815D2D90700A313B1 /* ThrowingFwrite.cpp in Sources */, + 1E365EEA16120F4600BA335B /* ChartTranslationOptions.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1338,6 +1498,7 @@ isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; COPY_PHASE_STRIP = NO; GCC_DYNAMIC_NO_PIC = NO; GCC_MODEL_TUNING = G5; @@ -1352,6 +1513,9 @@ "_FILE_OFFSET_BITS=64", _LARGE_FILES, WITH_THREADS, + IS_XCODE, + HAVE_CMPH, + "KENLM_MAX_ORDER=7", ); HEADER_SEARCH_PATHS = ( ../.., @@ -1376,6 +1540,7 @@ "\"$(SRCROOT)/../../moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi\"", ); PRODUCT_NAME = moses; + USER_HEADER_SEARCH_PATHS = "../.. ../../moses/src ../../irstlm/include ../../srilm/include ../../kenlm ../../randlm/include /opt/local/include ../../synlm/hhmm/wsjparse/include ../../synlm/hhmm/rvtl/include/ ../.. ../../cmph/include"; }; name = Debug; }; @@ -1383,6 +1548,7 @@ isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; GCC_MODEL_TUNING = G5; GCC_PREPROCESSOR_DEFINITIONS = ( @@ -1395,6 +1561,9 @@ "_FILE_OFFSET_BITS=64", _LARGE_FILES, WITH_THREADS, + IS_XCODE, + HAVE_CMPH, + "KENLM_MAX_ORDER=7", ); HEADER_SEARCH_PATHS = ( ../.., @@ -1419,6 +1588,7 @@ "\"$(SRCROOT)/../../moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi\"", ); PRODUCT_NAME = moses; + USER_HEADER_SEARCH_PATHS = "../.. ../../moses/src ../../irstlm/include ../../srilm/include ../../kenlm ../../randlm/include /opt/local/include ../../synlm/hhmm/wsjparse/include ../../synlm/hhmm/rvtl/include/ ../.. ../../cmph/include"; }; name = Release; }; diff --git a/contrib/other-builds/moses/.cproject b/contrib/other-builds/moses/.cproject index 2995d5eae..0148cc6f2 100644 --- a/contrib/other-builds/moses/.cproject +++ b/contrib/other-builds/moses/.cproject @@ -3,8 +3,8 @@ - - + + @@ -13,7 +13,7 @@ - + @@ -21,65 +21,70 @@ - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + - + - - + + - + @@ -88,59 +93,41 @@ - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + @@ -150,12 +137,24 @@ + + + + + + + + + + + + diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project index 8d534dbd4..31c11819a 100644 --- a/contrib/other-builds/moses/.project +++ b/contrib/other-builds/moses/.project @@ -101,6 +101,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/AlignmentInfoCollection.h + + ApplicableRuleTrie.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/ApplicableRuleTrie.cpp + + + ApplicableRuleTrie.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/ApplicableRuleTrie.h + BilingualDynSuffixArray.cpp 1 @@ -271,6 +281,11 @@ 1 PARENT-3-PROJECT_LOC/moses/src/ChartTrellisPathList.h + + CompactPT + 2 + PARENT-3-PROJECT_LOC/moses/src/CompactPT + ConfusionNet.cpp 1 @@ -441,6 +456,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/FloydWarshall.h + + FuzzyMatchWrapper.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/FuzzyMatchWrapper.cpp + + + FuzzyMatchWrapper.h + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/FuzzyMatchWrapper.h + GenerationDictionary.cpp 1 @@ -536,6 +561,11 @@ 1 PARENT-3-PROJECT_LOC/moses/src/InputType.h + + IntermediateVarSpanNode.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/IntermediateVarSpanNode.h + Jamfile 1 @@ -606,6 +636,11 @@ 1 PARENT-3-PROJECT_LOC/moses/src/Manager.h + + Match.h + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/Match.h + NonTerminal.cpp 1 @@ -661,6 +696,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/Parameter.h + + Parser.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/Parser.cpp + + + Parser.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/Parser.h + PartialTranslOptColl.cpp 1 @@ -809,7 +854,7 @@ RuleTable 2 - virtual:/virtual + PARENT-3-PROJECT_LOC/moses/src/RuleTable SRI.lo @@ -821,11 +866,6 @@ 1 PARENT-3-PROJECT_LOC/moses/src/SRI.o - - Scope3Parser - 2 - virtual:/virtual - ScoreComponentCollection.cpp 1 @@ -886,6 +926,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/SearchNormal.h + + SearchNormalBatch.cpp + 1 + PARENT-1-ECLIPSE_HOME/workspace/github/hieuhoang/moses/src/SearchNormalBatch.cpp + + + SearchNormalBatch.h + 1 + PARENT-1-ECLIPSE_HOME/workspace/github/hieuhoang/moses/src/SearchNormalBatch.h + Sentence.cpp 1 @@ -896,6 +946,21 @@ 1 PARENT-3-PROJECT_LOC/moses/src/Sentence.h + + SentenceAlignment.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/SentenceAlignment.cpp + + + SentenceAlignment.h + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/SentenceAlignment.h + + + SentenceMap.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/SentenceMap.h + SentenceStats.cpp 1 @@ -916,6 +981,26 @@ 1 PARENT-3-PROJECT_LOC/moses/src/SquareMatrix.h + + StackLattice.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLattice.h + + + StackLatticeBuilder.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLatticeBuilder.cpp + + + StackLatticeBuilder.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLatticeBuilder.h + + + StackLatticeSearcher.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLatticeSearcher.h + StackVec.h 1 @@ -941,6 +1026,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/StaticData.o + + SuffixArray.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/SuffixArray.cpp + + + SuffixArray.h + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/SuffixArray.h + SyntacticLanguageModel.cpp 1 @@ -1181,6 +1276,31 @@ 1 PARENT-3-PROJECT_LOC/moses/src/Util.o + + VarSpanNode.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/VarSpanNode.h + + + VarSpanTrieBuilder.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/VarSpanTrieBuilder.cpp + + + VarSpanTrieBuilder.h + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/VarSpanTrieBuilder.h + + + Vocabulary.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/Vocabulary.cpp + + + Vocabulary.h + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/Vocabulary.h + Word.cpp 1 @@ -1336,6 +1456,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/ChartRuleLookupManagerMemory.h + + CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp + + + CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h + CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp 1 @@ -1381,6 +1511,16 @@ 2 virtual:/virtual + + DynSAInclude/FileHandler.cpp + 1 + PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/FileHandler.cpp + + + DynSAInclude/FileHandler.h + 1 + PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/FileHandler.h + DynSAInclude/Jamfile 1 @@ -1396,26 +1536,11 @@ 1 PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/RandLMFilter.h - - DynSAInclude/bin - 2 - virtual:/virtual - DynSAInclude/fdstream.h 1 PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/fdstream.h - - DynSAInclude/file.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/file.cpp - - - DynSAInclude/file.h - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/file.h - DynSAInclude/hash.h 1 @@ -1616,211 +1741,16 @@ 2 virtual:/virtual - - RuleTable/Jamfile - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/Jamfile - - - RuleTable/Loader.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/Loader.h - - - RuleTable/LoaderCompact.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderCompact.cpp - - - RuleTable/LoaderCompact.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderCompact.h - - - RuleTable/LoaderFactory.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderFactory.cpp - - - RuleTable/LoaderFactory.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderFactory.h - - - RuleTable/LoaderHiero.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderHiero.cpp - - - RuleTable/LoaderHiero.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderHiero.h - - - RuleTable/LoaderStandard.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderStandard.cpp - - - RuleTable/LoaderStandard.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/LoaderStandard.h - - - RuleTable/PhraseDictionaryALSuffixArray.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionaryALSuffixArray.cpp - - - RuleTable/PhraseDictionaryALSuffixArray.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionaryALSuffixArray.h - - - RuleTable/PhraseDictionaryNodeSCFG.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionaryNodeSCFG.cpp - - - RuleTable/PhraseDictionaryNodeSCFG.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionaryNodeSCFG.h - - - RuleTable/PhraseDictionaryOnDisk.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionaryOnDisk.cpp - - - RuleTable/PhraseDictionaryOnDisk.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionaryOnDisk.h - - - RuleTable/PhraseDictionarySCFG.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionarySCFG.cpp - - - RuleTable/PhraseDictionarySCFG.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/PhraseDictionarySCFG.h - - - RuleTable/Trie.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/Trie.cpp - - - RuleTable/Trie.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/Trie.h - - - RuleTable/UTrie.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/UTrie.cpp - - - RuleTable/UTrie.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/UTrie.h - - - RuleTable/UTrieNode.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/UTrieNode.cpp - - - RuleTable/UTrieNode.h - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/UTrieNode.h - - - RuleTable/bin - 2 - virtual:/virtual - - - Scope3Parser/ApplicableRuleTrie.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/ApplicableRuleTrie.cpp - - - Scope3Parser/ApplicableRuleTrie.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/ApplicableRuleTrie.h - - - Scope3Parser/IntermediateVarSpanNode.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/IntermediateVarSpanNode.h - - - Scope3Parser/Jamfile - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/Jamfile - - - Scope3Parser/Parser.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/Parser.cpp - - - Scope3Parser/Parser.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/Parser.h - - - Scope3Parser/SentenceMap.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/SentenceMap.h - - - Scope3Parser/StackLattice.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLattice.h - - - Scope3Parser/StackLatticeBuilder.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLatticeBuilder.cpp - - - Scope3Parser/StackLatticeBuilder.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLatticeBuilder.h - - - Scope3Parser/StackLatticeSearcher.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/StackLatticeSearcher.h - - - Scope3Parser/VarSpanNode.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/VarSpanNode.h - - - Scope3Parser/VarSpanTrieBuilder.cpp - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/VarSpanTrieBuilder.cpp - - - Scope3Parser/VarSpanTrieBuilder.h - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/VarSpanTrieBuilder.h - - - Scope3Parser/bin - 2 - virtual:/virtual - bin/darwin-4.2.1 2 virtual:/virtual + + bin/gcc-4.6 + 2 + virtual:/virtual + CYKPlusParser/bin/clang-darwin-4.2.1 2 @@ -1832,12 +1762,7 @@ virtual:/virtual - DynSAInclude/bin/clang-darwin-4.2.1 - 2 - virtual:/virtual - - - DynSAInclude/bin/darwin-4.2.1 + CYKPlusParser/bin/gcc-4.6 2 virtual:/virtual @@ -1856,21 +1781,16 @@ 1 PARENT-3-PROJECT_LOC/moses/src/LM/bin/lm.log - - RuleTable/bin/darwin-4.2.1 - 2 - virtual:/virtual - - - Scope3Parser/bin/darwin-4.2.1 - 2 - virtual:/virtual - bin/darwin-4.2.1/release 2 virtual:/virtual + + bin/gcc-4.6/release + 2 + virtual:/virtual + CYKPlusParser/bin/clang-darwin-4.2.1/release 2 @@ -1882,12 +1802,7 @@ virtual:/virtual - DynSAInclude/bin/clang-darwin-4.2.1/release - 2 - virtual:/virtual - - - DynSAInclude/bin/darwin-4.2.1/release + CYKPlusParser/bin/gcc-4.6/release 2 virtual:/virtual @@ -1901,21 +1816,16 @@ 2 virtual:/virtual - - RuleTable/bin/darwin-4.2.1/release - 2 - virtual:/virtual - - - Scope3Parser/bin/darwin-4.2.1/release - 2 - virtual:/virtual - bin/darwin-4.2.1/release/debug-symbols-on 2 virtual:/virtual + + bin/gcc-4.6/release/debug-symbols-on + 2 + virtual:/virtual + CYKPlusParser/bin/clang-darwin-4.2.1/release/debug-symbols-on 2 @@ -1927,12 +1837,7 @@ virtual:/virtual - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on - 2 - virtual:/virtual - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on 2 virtual:/virtual @@ -1951,21 +1856,16 @@ 2 virtual:/virtual - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on - 2 - virtual:/virtual - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on - 2 - virtual:/virtual - bin/darwin-4.2.1/release/debug-symbols-on/link-static 2 virtual:/virtual + + bin/gcc-4.6/release/debug-symbols-on/link-static + 2 + virtual:/virtual + CYKPlusParser/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static 2 @@ -1982,12 +1882,7 @@ virtual:/virtual - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static - 2 - virtual:/virtual - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static 2 virtual:/virtual @@ -2011,31 +1906,16 @@ 2 virtual:/virtual - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static - 2 - virtual:/virtual - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi - 2 - virtual:/virtual - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static - 2 - virtual:/virtual - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi - 2 - virtual:/virtual - bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi 2 virtual:/virtual + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi + 2 + virtual:/virtual + CYKPlusParser/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi 2 @@ -2072,12 +1952,7 @@ PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/DotChartOnDisk.o - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi - 2 - virtual:/virtual - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi 2 virtual:/virtual @@ -2191,91 +2066,6 @@ 2 virtual:/virtual - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi - 2 - virtual:/virtual - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderCompact.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderCompact.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderFactory.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderFactory.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderHiero.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderHiero.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderStandard.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/LoaderStandard.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionaryALSuffixArray.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionaryALSuffixArray.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionaryNodeSCFG.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionaryNodeSCFG.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionaryOnDisk.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionaryOnDisk.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionarySCFG.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/PhraseDictionarySCFG.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/Trie.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/Trie.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/UTrie.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/UTrie.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/UTrieNode.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/UTrieNode.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi - 2 - virtual:/virtual - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/ApplicableRuleTrie.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/ApplicableRuleTrie.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/Parser.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/Parser.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/StackLatticeBuilder.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/StackLatticeBuilder.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/VarSpanTrieBuilder.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/threading-multi/VarSpanTrieBuilder.o - bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/AlignmentInfo.o 1 @@ -2751,6 +2541,56 @@ 1 PARENT-3-PROJECT_LOC/moses/src/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libmoses_internal.a + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ApplicableRuleTrie.o + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ApplicableRuleTrie.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/FuzzyMatchWrapper.o + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/FuzzyMatchWrapper.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/Parser.o + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/Parser.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/SentenceAlignment.o + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/SentenceAlignment.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/StackLatticeBuilder.o + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/StackLatticeBuilder.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/SuffixArray.o + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/SuffixArray.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/VarSpanTrieBuilder.o + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/VarSpanTrieBuilder.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/Vocabulary.o + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/Vocabulary.o + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/libScope3Parser.a + 1 + PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/libScope3Parser.a + + + bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/libfuzzy-match.a + 1 + PARENT-3-PROJECT_LOC/moses/src/fuzzy-match/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/libfuzzy-match.a + CYKPlusParser/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DotChartOnDisk.o 1 @@ -2787,24 +2627,39 @@ PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libCYKPlusParser.a - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude - 2 - virtual:/virtual - - - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libdynsa.a + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerCYKPlus.o 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libdynsa.a + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerCYKPlus.o - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude - 2 - virtual:/virtual - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libdynsa.a + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerMemory.o 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libdynsa.a + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerMemory.o + + + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerMemoryPerSentence.o + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerMemoryPerSentence.o + + + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerOnDisk.o + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/ChartRuleLookupManagerOnDisk.o + + + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/DotChartInMemory.o + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/DotChartInMemory.o + + + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/DotChartOnDisk.o + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/DotChartOnDisk.o + + + CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/libCYKPlusParser.a + 1 + PARENT-3-PROJECT_LOC/moses/src/CYKPlusParser/bin/gcc-4.6/release/debug-symbols-on/link-static/threading-multi/libCYKPlusParser.a LM/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/Base.o @@ -2921,91 +2776,6 @@ 1 PARENT-3-PROJECT_LOC/moses/src/LM/bin/gcc-4.2.1/release/debug-symbols-on/link-static/threading-multi/libLM.a - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderCompact.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderCompact.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderFactory.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderFactory.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderHiero.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderHiero.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderStandard.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/LoaderStandard.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryALSuffixArray.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryALSuffixArray.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryNodeSCFG.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryNodeSCFG.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryOnDisk.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionaryOnDisk.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionarySCFG.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/PhraseDictionarySCFG.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/Trie.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/Trie.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/UTrie.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/UTrie.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/UTrieNode.o - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/UTrieNode.o - - - RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libRuleTable.a - 1 - PARENT-3-PROJECT_LOC/moses/src/RuleTable/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libRuleTable.a - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ApplicableRuleTrie.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/ApplicableRuleTrie.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/Parser.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/Parser.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/StackLatticeBuilder.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/StackLatticeBuilder.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/VarSpanTrieBuilder.o - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/VarSpanTrieBuilder.o - - - Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libScope3Parser.a - 1 - PARENT-3-PROJECT_LOC/moses/src/Scope3Parser/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/libScope3Parser.a - bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/file.o 1 @@ -3021,35 +2791,5 @@ 1 PARENT-3-PROJECT_LOC/moses/src/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/vocab.o - - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/file.o - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/file.o - - - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/params.o - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/params.o - - - DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/vocab.o - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/clang-darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/vocab.o - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/file.o - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/file.o - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/params.o - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/params.o - - - DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/vocab.o - 1 - PARENT-3-PROJECT_LOC/moses/src/DynSAInclude/bin/darwin-4.2.1/release/debug-symbols-on/link-static/threading-multi/DynSAInclude/vocab.o - diff --git a/contrib/other-builds/mosesserver.vcxproj b/contrib/other-builds/mosesserver.vcxproj new file mode 100644 index 000000000..6d7470eec --- /dev/null +++ b/contrib/other-builds/mosesserver.vcxproj @@ -0,0 +1,102 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {85811FDF-8AD1-4490-A545-B2F51931A18C} + mosescmd + Win32Proj + + + + Application + Unicode + true + + + Application + Unicode + + + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + true + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + false + C:\Program Files\boost\boost_1_47;$(IncludePath) + C:\Program Files\boost\boost_1_47;$(IncludePath) + + + + Disabled + C:\xmlrpc-c\include;C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../..;%(AdditionalIncludeDirectories) + WITH_THREADS;NO_PIPES;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + EditAndContinue + + + libxmlrpc_server_abyss.lib;libxmlrpc_server.lib;libxmlrpc_abyss.lib;libxmlrpc.lib;libxmlrpc_util.lib;libxmlrpc_xmlparse.lib;libxmlrpc_xmltok.lib;libxmlrpc++.lib;C:\GnuWin32\lib\zlib.lib;$(SolutionDir)$(Configuration)\moses.lib;$(SolutionDir)$(Configuration)\kenlm.lib;$(SolutionDir)$(Configuration)\OnDiskPt.lib;%(AdditionalDependencies) + true + Console + false + + + MachineX86 + C:\xmlrpc-c\bin\Debug-Static-Win32;C:\boost\boost_1_47\lib + + + + + C:\xmlrpc-c\include;C:\boost\boost_1_47;$(SolutionDir)/../../moses/src;$(SolutionDir)/../..;%(AdditionalIncludeDirectories) + WITH_THREADS;NO_PIPES;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + MultiThreadedDLL + + + Level3 + ProgramDatabase + + + libxmlrpc_server_abyss.lib;libxmlrpc_server.lib;libxmlrpc_abyss.lib;libxmlrpc.lib;libxmlrpc_util.lib;libxmlrpc_xmlparse.lib;libxmlrpc_xmltok.lib;libxmlrpc++.lib;C:\GnuWin32\lib\zlib.lib;$(SolutionDir)$(Configuration)\moses.lib;$(SolutionDir)$(Configuration)\kenlm.lib;$(SolutionDir)$(Configuration)\OnDiskPt.lib;%(AdditionalDependencies) + true + Console + true + true + false + + + MachineX86 + C:\xmlrpc-c\bin\Release-Static-Win32;C:\boost\boost_1_47\lib + + + + + + + + + \ No newline at end of file diff --git a/contrib/other-builds/processLexicalTableMin.xcodeproj/project.pbxproj b/contrib/other-builds/processLexicalTableMin.xcodeproj/project.pbxproj new file mode 100644 index 000000000..113d9723d --- /dev/null +++ b/contrib/other-builds/processLexicalTableMin.xcodeproj/project.pbxproj @@ -0,0 +1,297 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 1E6D9FF115D027F00064D436 /* libmoses.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1EB3EBD515D0269B006B9CF1 /* libmoses.a */; }; + 1EB3EBB315D024C7006B9CF1 /* processLexicalTableMin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EB3EBB215D024C7006B9CF1 /* processLexicalTableMin.cpp */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 1E6D9FF215D0292D0064D436 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1EB3EBD015D0269B006B9CF1 /* moses.xcodeproj */; + proxyType = 1; + remoteGlobalIDString = D2AAC045055464E500DB518D; + remoteInfo = moses; + }; + 1EB3EBD415D0269B006B9CF1 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1EB3EBD015D0269B006B9CF1 /* moses.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = D2AAC046055464E500DB518D; + remoteInfo = moses; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 1E3A0AEA15D0242A003EF9B4 /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 1E3A0AEC15D0242A003EF9B4 /* processLexicalTableMin */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = processLexicalTableMin; sourceTree = BUILT_PRODUCTS_DIR; }; + 1EB3EBB215D024C7006B9CF1 /* processLexicalTableMin.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = processLexicalTableMin.cpp; path = ../../misc/processLexicalTableMin.cpp; sourceTree = ""; }; + 1EB3EBD015D0269B006B9CF1 /* moses.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; path = moses.xcodeproj; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 1E3A0AE915D0242A003EF9B4 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 1E6D9FF115D027F00064D436 /* libmoses.a in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 1E3A0AE115D02427003EF9B4 = { + isa = PBXGroup; + children = ( + 1EB3EBB215D024C7006B9CF1 /* processLexicalTableMin.cpp */, + 1E3A0AED15D0242A003EF9B4 /* Products */, + 1EB3EBD015D0269B006B9CF1 /* moses.xcodeproj */, + ); + sourceTree = ""; + }; + 1E3A0AED15D0242A003EF9B4 /* Products */ = { + isa = PBXGroup; + children = ( + 1E3A0AEC15D0242A003EF9B4 /* processLexicalTableMin */, + ); + name = Products; + sourceTree = ""; + }; + 1EB3EBD115D0269B006B9CF1 /* Products */ = { + isa = PBXGroup; + children = ( + 1EB3EBD515D0269B006B9CF1 /* libmoses.a */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 1E3A0AEB15D0242A003EF9B4 /* processLexicalTableMin */ = { + isa = PBXNativeTarget; + buildConfigurationList = 1E3A0AF615D0242B003EF9B4 /* Build configuration list for PBXNativeTarget "processLexicalTableMin" */; + buildPhases = ( + 1E3A0AE815D0242A003EF9B4 /* Sources */, + 1E3A0AE915D0242A003EF9B4 /* Frameworks */, + 1E3A0AEA15D0242A003EF9B4 /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + 1E6D9FF315D0292D0064D436 /* PBXTargetDependency */, + ); + name = processLexicalTableMin; + productName = processLexicalTableMin; + productReference = 1E3A0AEC15D0242A003EF9B4 /* processLexicalTableMin */; + productType = "com.apple.product-type.tool"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 1E3A0AE315D02427003EF9B4 /* Project object */ = { + isa = PBXProject; + buildConfigurationList = 1E3A0AE615D02427003EF9B4 /* Build configuration list for PBXProject "processLexicalTableMin" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + ); + mainGroup = 1E3A0AE115D02427003EF9B4; + productRefGroup = 1E3A0AED15D0242A003EF9B4 /* Products */; + projectDirPath = ""; + projectReferences = ( + { + ProductGroup = 1EB3EBD115D0269B006B9CF1 /* Products */; + ProjectRef = 1EB3EBD015D0269B006B9CF1 /* moses.xcodeproj */; + }, + ); + projectRoot = ""; + targets = ( + 1E3A0AEB15D0242A003EF9B4 /* processLexicalTableMin */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXReferenceProxy section */ + 1EB3EBD515D0269B006B9CF1 /* libmoses.a */ = { + isa = PBXReferenceProxy; + fileType = archive.ar; + path = libmoses.a; + remoteRef = 1EB3EBD415D0269B006B9CF1 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; +/* End PBXReferenceProxy section */ + +/* Begin PBXSourcesBuildPhase section */ + 1E3A0AE815D0242A003EF9B4 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 1EB3EBB315D024C7006B9CF1 /* processLexicalTableMin.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 1E6D9FF315D0292D0064D436 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = moses; + targetProxy = 1E6D9FF215D0292D0064D436 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 1E3A0AF415D0242B003EF9B4 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + COPY_PHASE_STRIP = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_SYMBOLS_PRIVATE_EXTERN = NO; + GCC_VERSION = com.apple.compilers.llvm.clang.1_0; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = ( + ../../, + ../../irstlm/include, + /opt/local/include, + ); + MACOSX_DEPLOYMENT_TARGET = 10.7; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + USER_HEADER_SEARCH_PATHS = "../../ ../../irstlm/include /opt/local/include ../../moses/src"; + }; + name = Debug; + }; + 1E3A0AF515D0242B003EF9B4 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + COPY_PHASE_STRIP = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_VERSION = com.apple.compilers.llvm.clang.1_0; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = ( + ../../, + ../../irstlm/include, + /opt/local/include, + ); + MACOSX_DEPLOYMENT_TARGET = 10.7; + SDKROOT = macosx; + USER_HEADER_SEARCH_PATHS = "../../ ../../irstlm/include /opt/local/include ../../moses/src"; + }; + name = Release; + }; + 1E3A0AF715D0242B003EF9B4 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_PREPROCESSOR_DEFINITIONS = WITH_THREADS; + "GCC_PREPROCESSOR_DEFINITIONS[arch=*]" = WITH_THREADS; + LIBRARY_SEARCH_PATHS = ( + ../../irstlm/lib, + ../../srilm/lib/macosx, + ../../randlm/lib, + /opt/local/lib, + ); + OTHER_LDFLAGS = ( + "-lz", + "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", + "-lrandlm", + "-lboost_thread-mt", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 1E3A0AF815D0242B003EF9B4 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_PREPROCESSOR_DEFINITIONS = WITH_THREADS; + LIBRARY_SEARCH_PATHS = ( + ../../irstlm/lib, + ../../srilm/lib/macosx, + ../../randlm/lib, + /opt/local/lib, + ); + OTHER_LDFLAGS = ( + "-lz", + "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", + "-lrandlm", + "-lboost_thread-mt", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 1E3A0AE615D02427003EF9B4 /* Build configuration list for PBXProject "processLexicalTableMin" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1E3A0AF415D0242B003EF9B4 /* Debug */, + 1E3A0AF515D0242B003EF9B4 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 1E3A0AF615D0242B003EF9B4 /* Build configuration list for PBXNativeTarget "processLexicalTableMin" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1E3A0AF715D0242B003EF9B4 /* Debug */, + 1E3A0AF815D0242B003EF9B4 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 1E3A0AE315D02427003EF9B4 /* Project object */; +} diff --git a/contrib/other-builds/processPhraseTableMin.xcodeproj/project.pbxproj b/contrib/other-builds/processPhraseTableMin.xcodeproj/project.pbxproj new file mode 100644 index 000000000..9db1d49b8 --- /dev/null +++ b/contrib/other-builds/processPhraseTableMin.xcodeproj/project.pbxproj @@ -0,0 +1,304 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 1EF3D68A15D02AEF00969478 /* processPhraseTableMin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EF3D68915D02AEF00969478 /* processPhraseTableMin.cpp */; }; + 1EF3D6A415D02B6400969478 /* libmoses.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1EF3D69915D02B4400969478 /* libmoses.a */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 1EF3D69815D02B4400969478 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1EF3D69415D02B4400969478 /* moses.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = D2AAC046055464E500DB518D; + remoteInfo = moses; + }; + 1EF3D6A515D02B6B00969478 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1EF3D69415D02B4400969478 /* moses.xcodeproj */; + proxyType = 1; + remoteGlobalIDString = D2AAC045055464E500DB518D; + remoteInfo = moses; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 1E6D9FFD15D02A8D0064D436 /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 1E6D9FFF15D02A8D0064D436 /* processPhraseTableMin */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = processPhraseTableMin; sourceTree = BUILT_PRODUCTS_DIR; }; + 1EF3D68915D02AEF00969478 /* processPhraseTableMin.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = processPhraseTableMin.cpp; path = ../../misc/processPhraseTableMin.cpp; sourceTree = ""; }; + 1EF3D69415D02B4400969478 /* moses.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; path = moses.xcodeproj; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 1E6D9FFC15D02A8D0064D436 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 1EF3D6A415D02B6400969478 /* libmoses.a in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 1E6D9FF415D02A8C0064D436 = { + isa = PBXGroup; + children = ( + 1EF3D68915D02AEF00969478 /* processPhraseTableMin.cpp */, + 1E6DA00015D02A8D0064D436 /* Products */, + 1EF3D69415D02B4400969478 /* moses.xcodeproj */, + ); + sourceTree = ""; + }; + 1E6DA00015D02A8D0064D436 /* Products */ = { + isa = PBXGroup; + children = ( + 1E6D9FFF15D02A8D0064D436 /* processPhraseTableMin */, + ); + name = Products; + sourceTree = ""; + }; + 1EF3D69515D02B4400969478 /* Products */ = { + isa = PBXGroup; + children = ( + 1EF3D69915D02B4400969478 /* libmoses.a */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 1E6D9FFE15D02A8D0064D436 /* processPhraseTableMin */ = { + isa = PBXNativeTarget; + buildConfigurationList = 1E6DA00915D02A8D0064D436 /* Build configuration list for PBXNativeTarget "processPhraseTableMin" */; + buildPhases = ( + 1E6D9FFB15D02A8D0064D436 /* Sources */, + 1E6D9FFC15D02A8D0064D436 /* Frameworks */, + 1E6D9FFD15D02A8D0064D436 /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + 1EF3D6A615D02B6B00969478 /* PBXTargetDependency */, + ); + name = processPhraseTableMin; + productName = processPhraseTableMin; + productReference = 1E6D9FFF15D02A8D0064D436 /* processPhraseTableMin */; + productType = "com.apple.product-type.tool"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 1E6D9FF615D02A8C0064D436 /* Project object */ = { + isa = PBXProject; + buildConfigurationList = 1E6D9FF915D02A8C0064D436 /* Build configuration list for PBXProject "processPhraseTableMin" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + ); + mainGroup = 1E6D9FF415D02A8C0064D436; + productRefGroup = 1E6DA00015D02A8D0064D436 /* Products */; + projectDirPath = ""; + projectReferences = ( + { + ProductGroup = 1EF3D69515D02B4400969478 /* Products */; + ProjectRef = 1EF3D69415D02B4400969478 /* moses.xcodeproj */; + }, + ); + projectRoot = ""; + targets = ( + 1E6D9FFE15D02A8D0064D436 /* processPhraseTableMin */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXReferenceProxy section */ + 1EF3D69915D02B4400969478 /* libmoses.a */ = { + isa = PBXReferenceProxy; + fileType = archive.ar; + path = libmoses.a; + remoteRef = 1EF3D69815D02B4400969478 /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; +/* End PBXReferenceProxy section */ + +/* Begin PBXSourcesBuildPhase section */ + 1E6D9FFB15D02A8D0064D436 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 1EF3D68A15D02AEF00969478 /* processPhraseTableMin.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 1EF3D6A615D02B6B00969478 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = moses; + targetProxy = 1EF3D6A515D02B6B00969478 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 1E6DA00715D02A8D0064D436 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + COPY_PHASE_STRIP = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_SYMBOLS_PRIVATE_EXTERN = NO; + GCC_VERSION = com.apple.compilers.llvm.clang.1_0; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + LIBRARY_SEARCH_PATHS = ""; + MACOSX_DEPLOYMENT_TARGET = 10.7; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + }; + name = Debug; + }; + 1E6DA00815D02A8D0064D436 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + COPY_PHASE_STRIP = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_VERSION = com.apple.compilers.llvm.clang.1_0; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + LIBRARY_SEARCH_PATHS = ""; + MACOSX_DEPLOYMENT_TARGET = 10.7; + SDKROOT = macosx; + }; + name = Release; + }; + 1E6DA00A15D02A8D0064D436 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_PREPROCESSOR_DEFINITIONS = WITH_THREADS; + HEADER_SEARCH_PATHS = ( + ../../, + ../../irstlm/include, + /opt/local/include, + ../../moses/src, + ../../cmph/include, + ); + LIBRARY_SEARCH_PATHS = ( + ../../irstlm/lib, + ../../srilm/lib/macosx, + ../../randlm/lib, + /opt/local/lib, + ../../cmph/lib, + ); + OTHER_LDFLAGS = ( + "-lz", + "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", + "-lrandlm", + "-lboost_thread-mt", + "-lcmph", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 1E6DA00B15D02A8D0064D436 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + GCC_PREPROCESSOR_DEFINITIONS = WITH_THREADS; + HEADER_SEARCH_PATHS = ( + ../../, + ../../irstlm/include, + /opt/local/include, + ../../moses/src, + ../../cmph/include, + ); + LIBRARY_SEARCH_PATHS = ( + ../../irstlm/lib, + ../../srilm/lib/macosx, + ../../randlm/lib, + /opt/local/lib, + ../../cmph/lib, + ); + OTHER_LDFLAGS = ( + "-lz", + "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", + "-lrandlm", + "-lboost_thread-mt", + "-lcmph", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 1E6D9FF915D02A8C0064D436 /* Build configuration list for PBXProject "processPhraseTableMin" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1E6DA00715D02A8D0064D436 /* Debug */, + 1E6DA00815D02A8D0064D436 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 1E6DA00915D02A8D0064D436 /* Build configuration list for PBXNativeTarget "processPhraseTableMin" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1E6DA00A15D02A8D0064D436 /* Debug */, + 1E6DA00B15D02A8D0064D436 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 1E6D9FF615D02A8C0064D436 /* Project object */; +} diff --git a/contrib/other-builds/util/.cproject b/contrib/other-builds/util/.cproject index 46e9a02b6..8ea5ab73b 100644 --- a/contrib/other-builds/util/.cproject +++ b/contrib/other-builds/util/.cproject @@ -41,9 +41,12 @@ + @@ -130,4 +133,5 @@ + diff --git a/contrib/python/README.md b/contrib/python/README.md new file mode 100644 index 000000000..fa7d270c8 --- /dev/null +++ b/contrib/python/README.md @@ -0,0 +1,28 @@ +# Python interface to Moses + +The idea is to have some of Moses' internals exposed to Python (inspired on pycdec). + +## What's been interfaced? + +* Binary phrase table: + + Moses::PhraseDictionaryTree.h + +## Building + +1. Build the python extension + + python setup.py build_ext -i [--with-cmph] + +3. Check the example code + + echo "casa" | python example.py examples/phrase-table 5 1 + echo "essa casa" | python example.py examples/phrase-table 5 1 + +## Changing the code + +If you want to add your changes you are going to have to recompile the cython code. + +1. Compile the cython code (use Cython 0.16): this will generate binpt/binpt.cpp + + cython --cplus binpt/binpt.pyx diff --git a/contrib/python/binpt/binpt.cpp b/contrib/python/binpt/binpt.cpp new file mode 100644 index 000000000..7de3058fc --- /dev/null +++ b/contrib/python/binpt/binpt.cpp @@ -0,0 +1,5648 @@ +/* Generated by Cython 0.16 on Tue Sep 18 11:36:58 2012 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02040000 + #error Cython requires Python 2.4+. +#else +#include /* For offsetof */ +#ifndef offsetof +#define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif + +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif + +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif + +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif + +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif + +#ifdef PYPY_VERSION +#define CYTHON_COMPILING_IN_PYPY 1 +#define CYTHON_COMPILING_IN_CPYTHON 0 +#else +#define CYTHON_COMPILING_IN_PYPY 0 +#define CYTHON_COMPILING_IN_CPYTHON 1 +#endif + +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyCFunction_Call PyObject_Call +#else + #define __Pyx_PyCFunction_Call PyCFunction_Call +#endif + +#if PY_VERSION_HEX < 0x02050000 + typedef int Py_ssize_t; + #define PY_SSIZE_T_MAX INT_MAX + #define PY_SSIZE_T_MIN INT_MIN + #define PY_FORMAT_SIZE_T "" + #define PyInt_FromSsize_t(z) PyInt_FromLong(z) + #define PyInt_AsSsize_t(o) __Pyx_PyInt_AsInt(o) + #define PyNumber_Index(o) PyNumber_Int(o) + #define PyIndex_Check(o) PyNumber_Check(o) + #define PyErr_WarnEx(category, message, stacklevel) PyErr_Warn(category, message) + #define __PYX_BUILD_PY_SSIZE_T "i" +#else + #define __PYX_BUILD_PY_SSIZE_T "n" +#endif + +#if PY_VERSION_HEX < 0x02060000 + #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) + #define PyVarObject_HEAD_INIT(type, size) \ + PyObject_HEAD_INIT(type) size, + #define PyType_Modified(t) + + typedef struct { + void *buf; + PyObject *obj; + Py_ssize_t len; + Py_ssize_t itemsize; + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; + } Py_buffer; + + #define PyBUF_SIMPLE 0 + #define PyBUF_WRITABLE 0x0001 + #define PyBUF_FORMAT 0x0004 + #define PyBUF_ND 0x0008 + #define PyBUF_STRIDES (0x0010 | PyBUF_ND) + #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) + #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) + #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) + #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + #define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_FORMAT | PyBUF_WRITABLE) + #define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_FORMAT | PyBUF_WRITABLE) + + typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); + typedef void (*releasebufferproc)(PyObject *, Py_buffer *); +#endif + +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif + +#if PY_MAJOR_VERSION < 3 && PY_MINOR_VERSION < 6 + #define PyUnicode_FromString(s) PyUnicode_Decode(s, strlen(s), "UTF-8", "strict") +#endif + +#if PY_MAJOR_VERSION >= 3 + #define Py_TPFLAGS_CHECKTYPES 0 + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif + +#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif + + +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_GET_LENGTH) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) +#else + #define CYTHON_PEP393_ENABLED 0 + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) +#endif + +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif + +#if PY_VERSION_HEX < 0x02060000 + #define PyBytesObject PyStringObject + #define PyBytes_Type PyString_Type + #define PyBytes_Check PyString_Check + #define PyBytes_CheckExact PyString_CheckExact + #define PyBytes_FromString PyString_FromString + #define PyBytes_FromStringAndSize PyString_FromStringAndSize + #define PyBytes_FromFormat PyString_FromFormat + #define PyBytes_DecodeEscape PyString_DecodeEscape + #define PyBytes_AsString PyString_AsString + #define PyBytes_AsStringAndSize PyString_AsStringAndSize + #define PyBytes_Size PyString_Size + #define PyBytes_AS_STRING PyString_AS_STRING + #define PyBytes_GET_SIZE PyString_GET_SIZE + #define PyBytes_Repr PyString_Repr + #define PyBytes_Concat PyString_Concat + #define PyBytes_ConcatAndDel PyString_ConcatAndDel +#endif + +#if PY_VERSION_HEX < 0x02060000 + #define PySet_Check(obj) PyObject_TypeCheck(obj, &PySet_Type) + #define PyFrozenSet_Check(obj) PyObject_TypeCheck(obj, &PyFrozenSet_Type) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif + +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) + +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask +#endif + +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif + +#if PY_VERSION_HEX < 0x03020000 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif + +#if (PY_MAJOR_VERSION < 3) || (PY_VERSION_HEX >= 0x03010300) + #define __Pyx_PySequence_GetSlice(obj, a, b) PySequence_GetSlice(obj, a, b) + #define __Pyx_PySequence_SetSlice(obj, a, b, value) PySequence_SetSlice(obj, a, b, value) + #define __Pyx_PySequence_DelSlice(obj, a, b) PySequence_DelSlice(obj, a, b) +#else + #define __Pyx_PySequence_GetSlice(obj, a, b) (unlikely(!(obj)) ? \ + (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), (PyObject*)0) : \ + (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_GetSlice(obj, a, b)) : \ + (PyErr_Format(PyExc_TypeError, "'%.200s' object is unsliceable", (obj)->ob_type->tp_name), (PyObject*)0))) + #define __Pyx_PySequence_SetSlice(obj, a, b, value) (unlikely(!(obj)) ? \ + (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \ + (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_SetSlice(obj, a, b, value)) : \ + (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice assignment", (obj)->ob_type->tp_name), -1))) + #define __Pyx_PySequence_DelSlice(obj, a, b) (unlikely(!(obj)) ? \ + (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \ + (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_DelSlice(obj, a, b)) : \ + (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice deletion", (obj)->ob_type->tp_name), -1))) +#endif + +#if PY_MAJOR_VERSION >= 3 + #define PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#endif + +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) +#else + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) +#endif + +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_NAMESTR(n) ((char *)(n)) + #define __Pyx_DOCSTR(n) ((char *)(n)) +#else + #define __Pyx_NAMESTR(n) (n) + #define __Pyx_DOCSTR(n) (n) +#endif + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#if defined(WIN32) || defined(MS_WINDOWS) +#define _USE_MATH_DEFINES +#endif +#include +#define __PYX_HAVE__binpt +#define __PYX_HAVE_API__binpt +#include +#include +#include +#include "TypeDef.h" +#include "PhraseDictionaryTree.h" +#include "Util.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + + +/* inline attribute */ +#ifndef CYTHON_INLINE + #if defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif + +/* unused attribute */ +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif + +typedef struct {PyObject **p; char *s; const long n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/ + + +/* Type Conversion Predeclarations */ + +#define __Pyx_PyBytes_FromUString(s) PyBytes_FromString((char*)s) +#define __Pyx_PyBytes_AsUString(s) ((unsigned char*) PyBytes_AsString(s)) + +#define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); + +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); + +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) + +#ifdef __GNUC__ + /* Test for GCC > 2.95 */ + #if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) + #else /* __GNUC__ > 2 ... */ + #define likely(x) (x) + #define unlikely(x) (x) + #endif /* __GNUC__ > 2 ... */ +#else /* __GNUC__ */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "binpt.pyx", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_5binpt_QueryResult; +struct __pyx_obj_5binpt_BinaryPhraseTable; +struct __pyx_opt_args_5binpt_get_query_result; + +/* "binpt.pxd":5 + * from libcpp.pair cimport pair + * + * ctypedef string* str_pointer # <<<<<<<<<<<<<< + * + * cdef extern from 'TypeDef.h' namespace 'Moses': + */ +typedef std::string *__pyx_t_5binpt_str_pointer; + +/* "binpt.pyx":71 + * return repr((repr(self._words), repr(self._scores), repr(self._wa))) + * + * cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None): # <<<<<<<<<<<<<< + * '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string) + * to a QueryResult (python object).''' + */ +struct __pyx_opt_args_5binpt_get_query_result { + int __pyx_n; + PyObject *wa; +}; + +/* "binpt.pyx":17 + * raise TypeError('Cannot convert %s to string' % type(data)) + * + * cdef class QueryResult(object): # <<<<<<<<<<<<<< + * '''This class represents a query result, that is, + * a target phrase (tuple of words/strings), + */ +struct __pyx_obj_5binpt_QueryResult { + PyObject_HEAD + PyObject *_words; + PyObject *_scores; + PyObject *_wa; +}; + + +/* "binpt.pyx":78 + * return QueryResult(words, scores, wa) + * + * cdef class BinaryPhraseTable(object): # <<<<<<<<<<<<<< + * '''This class encapsulates a Moses::PhraseDictionaryTree for operations over + * binary phrase tables.''' + */ +struct __pyx_obj_5binpt_BinaryPhraseTable { + PyObject_HEAD + Moses::PhraseDictionaryTree *__pyx___tree; + PyObject *_path; + unsigned int _nscores; + int _wa; + PyObject *_delimiters; +}; + +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); /*proto*/ + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil) \ + if (acquire_gil) { \ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + PyGILState_Release(__pyx_gilstate_save); \ + } else { \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil) \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext() \ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif /* CYTHON_REFNANNY */ +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ + +static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ +static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); /*proto*/ + +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ + +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); /*proto*/ + +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ + const char* function_name); /*proto*/ + +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (!j) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +#define __Pyx_GetItemInt_List(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \ + __Pyx_GetItemInt_List_Fast(o, i) : \ + __Pyx_GetItemInt_Generic(o, to_py_func(i))) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i) { + if (likely(o != Py_None)) { + if (likely((0 <= i) & (i < PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, i); + Py_INCREF(r); + return r; + } + else if ((-PyList_GET_SIZE(o) <= i) & (i < 0)) { + PyObject *r = PyList_GET_ITEM(o, PyList_GET_SIZE(o) + i); + Py_INCREF(r); + return r; + } + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} +#define __Pyx_GetItemInt_Tuple(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \ + __Pyx_GetItemInt_Tuple_Fast(o, i) : \ + __Pyx_GetItemInt_Generic(o, to_py_func(i))) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i) { + if (likely(o != Py_None)) { + if (likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, i); + Py_INCREF(r); + return r; + } + else if ((-PyTuple_GET_SIZE(o) <= i) & (i < 0)) { + PyObject *r = PyTuple_GET_ITEM(o, PyTuple_GET_SIZE(o) + i); + Py_INCREF(r); + return r; + } + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} +#define __Pyx_GetItemInt(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \ + __Pyx_GetItemInt_Fast(o, i) : \ + __Pyx_GetItemInt_Generic(o, to_py_func(i))) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i) { + if (PyList_CheckExact(o)) { + Py_ssize_t n = (likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if (likely((n >= 0) & (n < PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = (likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if (likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } + else if (likely(i >= 0)) { + PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; + if (likely(m && m->sq_item)) { + return m->sq_item(o, i); + } + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} + +static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact); /*proto*/ + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level); /*proto*/ + +#define __Pyx_CyFunction_USED 1 +#include +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CyFunction_GetClosure(f) \ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#define __Pyx_CyFunction_GetClassObj(f) \ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#define __Pyx_CyFunction_Defaults(type, f) \ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g) \ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { + PyCFunctionObject func; + int flags; + PyObject *func_dict; + PyObject *func_weakreflist; + PyObject *func_name; + PyObject *func_doc; + PyObject *func_code; + PyObject *func_closure; + PyObject *func_classobj; /* No-args super() class cell */ + void *defaults; + int defaults_pyobjects; + PyObject *defaults_tuple; /* Const defaults tuple */ + PyObject *(*defaults_getter)(PyObject *); +} __pyx_CyFunctionObject; +static PyTypeObject *__pyx_CyFunctionType = 0; +#define __Pyx_CyFunction_NewEx(ml, flags, self, module, code) \ + __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, self, module, code) +static PyObject *__Pyx_CyFunction_New(PyTypeObject *, + PyMethodDef *ml, int flags, + PyObject *self, PyObject *module, + PyObject* code); +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, + size_t size, + int pyobjects); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static int __Pyx_CyFunction_init(void); + +static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); + +static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); + +static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); + +static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject *); + +static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject *); + +static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject *); + +static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); + +static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); + +static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); + +static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject *); + +static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); + +static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); + +static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject *); + +static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); + +static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); + +static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); + +static int __Pyx_check_binary_version(void); + +typedef struct { + int code_line; + PyCodeObject* code_object; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); /*proto*/ + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ + + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libcpp.utility' */ + +/* Module declarations from 'libcpp.pair' */ + +/* Module declarations from 'cython' */ + +/* Module declarations from 'binpt' */ +static PyTypeObject *__pyx_ptype_5binpt_QueryResult = 0; +static PyTypeObject *__pyx_ptype_5binpt_BinaryPhraseTable = 0; +static int __pyx_f_5binpt_fsign(float, int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_5binpt_as_str(PyObject *); /*proto*/ +static struct __pyx_obj_5binpt_QueryResult *__pyx_f_5binpt_get_query_result(Moses::StringTgtCand &, struct __pyx_opt_args_5binpt_get_query_result *__pyx_optional_args); /*proto*/ +#define __Pyx_MODULE_NAME "binpt" +int __pyx_module_is_main_binpt = 0; + +/* Implementation of 'binpt' */ +static PyObject *__pyx_builtin_property; +static PyObject *__pyx_builtin_staticmethod; +static PyObject *__pyx_builtin_TypeError; +static PyObject *__pyx_builtin_range; +static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_pf_5binpt_fsign(CYTHON_UNUSED PyObject *__pyx_self, float __pyx_v_x); /* proto */ +static int __pyx_pf_5binpt_11QueryResult___cinit__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self, PyObject *__pyx_v_words, PyObject *__pyx_v_scores, PyObject *__pyx_v_wa); /* proto */ +static PyObject *__pyx_pf_5binpt_11QueryResult_2words(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_11QueryResult_4scores(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_11QueryResult_6wa(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda1(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_r); /* proto */ +static PyObject *__pyx_pf_5binpt_11QueryResult_8desc(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_keys); /* proto */ +static PyObject *__pyx_pf_5binpt_11QueryResult_10__str__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_11QueryResult_12__repr__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */ +static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_path, unsigned int __pyx_v_nscores, int __pyx_v_wa, PyObject *__pyx_v_delimiters); /* proto */ +static void __pyx_pf_5binpt_17BinaryPhraseTable_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObject *__pyx_v_stem, int __pyx_v_wa); /* proto */ +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_6path(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_8nscores(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_10wa(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiters(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_line, PyObject *__pyx_v_cmp, PyObject *__pyx_v_top); /* proto */ +static char __pyx_k_1[] = "UTF-8"; +static char __pyx_k_3[] = "Cannot convert %s to string"; +static char __pyx_k_5[] = " ||| "; +static char __pyx_k_6[] = " "; +static char __pyx_k_7[] = " \t"; +static char __pyx_k_8[] = "'%s' doesn't seem a valid binary table."; +static char __pyx_k_9[] = ".binphr.idx"; +static char __pyx_k_10[] = ".binphr.srctree.wa"; +static char __pyx_k_11[] = ".binphr.srcvoc"; +static char __pyx_k_12[] = ".binphr.tgtdata.wa"; +static char __pyx_k_13[] = ".binphr.tgtvoc"; +static char __pyx_k_14[] = ".binphr.srctree"; +static char __pyx_k_15[] = ".binphr.tgtdata"; +static char __pyx_k_18[] = "/media/Data/tools/moses/mosesdecoder/contrib/python/binpt/binpt.pyx"; +static char __pyx_k__x[] = "x"; +static char __pyx_k__y[] = "y"; +static char __pyx_k__os[] = "os"; +static char __pyx_k__wa[] = "wa"; +static char __pyx_k__cmp[] = "cmp"; +static char __pyx_k__top[] = "top"; +static char __pyx_k__desc[] = "desc"; +static char __pyx_k__join[] = "join"; +static char __pyx_k__keys[] = "keys"; +static char __pyx_k__line[] = "line"; +static char __pyx_k__path[] = "path"; +static char __pyx_k__sort[] = "sort"; +static char __pyx_k__stem[] = "stem"; +static char __pyx_k__binpt[] = "binpt"; +static char __pyx_k__range[] = "range"; +static char __pyx_k__words[] = "words"; +static char __pyx_k__encode[] = "encode"; +static char __pyx_k__isfile[] = "isfile"; +static char __pyx_k__scores[] = "scores"; +static char __pyx_k__nscores[] = "nscores"; +static char __pyx_k____main__[] = "__main__"; +static char __pyx_k____test__[] = "__test__"; +static char __pyx_k__property[] = "property"; +static char __pyx_k__TypeError[] = "TypeError"; +static char __pyx_k__ValueError[] = "ValueError"; +static char __pyx_k__delimiters[] = "delimiters"; +static char __pyx_k__staticmethod[] = "staticmethod"; +static char __pyx_k__isValidBinaryTable[] = "isValidBinaryTable"; +static PyObject *__pyx_kp_s_1; +static PyObject *__pyx_kp_s_10; +static PyObject *__pyx_kp_s_11; +static PyObject *__pyx_kp_s_12; +static PyObject *__pyx_kp_s_13; +static PyObject *__pyx_kp_s_14; +static PyObject *__pyx_kp_s_15; +static PyObject *__pyx_kp_s_18; +static PyObject *__pyx_kp_s_3; +static PyObject *__pyx_kp_s_5; +static PyObject *__pyx_kp_s_6; +static PyObject *__pyx_kp_s_7; +static PyObject *__pyx_kp_s_8; +static PyObject *__pyx_kp_s_9; +static PyObject *__pyx_n_s__TypeError; +static PyObject *__pyx_n_s__ValueError; +static PyObject *__pyx_n_s____main__; +static PyObject *__pyx_n_s____test__; +static PyObject *__pyx_n_s__binpt; +static PyObject *__pyx_n_s__cmp; +static PyObject *__pyx_n_s__delimiters; +static PyObject *__pyx_n_s__desc; +static PyObject *__pyx_n_s__encode; +static PyObject *__pyx_n_s__isValidBinaryTable; +static PyObject *__pyx_n_s__isfile; +static PyObject *__pyx_n_s__join; +static PyObject *__pyx_n_s__keys; +static PyObject *__pyx_n_s__line; +static PyObject *__pyx_n_s__nscores; +static PyObject *__pyx_n_s__os; +static PyObject *__pyx_n_s__path; +static PyObject *__pyx_n_s__property; +static PyObject *__pyx_n_s__range; +static PyObject *__pyx_n_s__scores; +static PyObject *__pyx_n_s__sort; +static PyObject *__pyx_n_s__staticmethod; +static PyObject *__pyx_n_s__stem; +static PyObject *__pyx_n_s__top; +static PyObject *__pyx_n_s__wa; +static PyObject *__pyx_n_s__words; +static PyObject *__pyx_n_s__x; +static PyObject *__pyx_n_s__y; +static PyObject *__pyx_int_0; +static PyObject *__pyx_k_4; +static PyObject *__pyx_k_tuple_2; +static PyObject *__pyx_k_tuple_16; +static PyObject *__pyx_k_tuple_19; +static PyObject *__pyx_k_codeobj_17; +static PyObject *__pyx_k_codeobj_20; + +/* "binpt.pyx":6 + * import cython + * + * cpdef int fsign(float x): # <<<<<<<<<<<<<< + * '''Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing''' + * return 1 if x >= 0 else -1 + */ + +static PyObject *__pyx_pw_5binpt_1fsign(PyObject *__pyx_self, PyObject *__pyx_arg_x); /*proto*/ +static int __pyx_f_5binpt_fsign(float __pyx_v_x, CYTHON_UNUSED int __pyx_skip_dispatch) { + int __pyx_r; + __Pyx_RefNannyDeclarations + long __pyx_t_1; + __Pyx_RefNannySetupContext("fsign", 0); + + /* "binpt.pyx":8 + * cpdef int fsign(float x): + * '''Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing''' + * return 1 if x >= 0 else -1 # <<<<<<<<<<<<<< + * + * cdef bytes as_str(data): + */ + if ((__pyx_v_x >= 0.0)) { + __pyx_t_1 = 1; + } else { + __pyx_t_1 = -1; + } + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + __pyx_r = 0; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_1fsign(PyObject *__pyx_self, PyObject *__pyx_arg_x); /*proto*/ +static char __pyx_doc_5binpt_fsign[] = "Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing"; +static PyObject *__pyx_pw_5binpt_1fsign(PyObject *__pyx_self, PyObject *__pyx_arg_x) { + float __pyx_v_x; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("fsign (wrapper)", 0); + __pyx_self = __pyx_self; + assert(__pyx_arg_x); { + __pyx_v_x = __pyx_PyFloat_AsFloat(__pyx_arg_x); if (unlikely((__pyx_v_x == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + __Pyx_AddTraceback("binpt.fsign", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5binpt_fsign(__pyx_self, ((float)__pyx_v_x)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":6 + * import cython + * + * cpdef int fsign(float x): # <<<<<<<<<<<<<< + * '''Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing''' + * return 1 if x >= 0 else -1 + */ + +static PyObject *__pyx_pf_5binpt_fsign(CYTHON_UNUSED PyObject *__pyx_self, float __pyx_v_x) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("fsign", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyInt_FromLong(__pyx_f_5binpt_fsign(__pyx_v_x, 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("binpt.fsign", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":10 + * return 1 if x >= 0 else -1 + * + * cdef bytes as_str(data): # <<<<<<<<<<<<<< + * if isinstance(data, bytes): + * return data + */ + +static PyObject *__pyx_f_5binpt_as_str(PyObject *__pyx_v_data) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("as_str", 0); + + /* "binpt.pyx":11 + * + * cdef bytes as_str(data): + * if isinstance(data, bytes): # <<<<<<<<<<<<<< + * return data + * elif isinstance(data, unicode): + */ + __pyx_t_1 = ((PyObject *)((PyObject*)(&PyBytes_Type))); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_data, __pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "binpt.pyx":12 + * cdef bytes as_str(data): + * if isinstance(data, bytes): + * return data # <<<<<<<<<<<<<< + * elif isinstance(data, unicode): + * return data.encode('UTF-8') + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + if (!(likely(PyBytes_CheckExact(__pyx_v_data))||((__pyx_v_data) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_data)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_v_data); + __pyx_r = ((PyObject*)__pyx_v_data); + goto __pyx_L0; + goto __pyx_L3; + } + + /* "binpt.pyx":13 + * if isinstance(data, bytes): + * return data + * elif isinstance(data, unicode): # <<<<<<<<<<<<<< + * return data.encode('UTF-8') + * raise TypeError('Cannot convert %s to string' % type(data)) + */ + __pyx_t_1 = ((PyObject *)((PyObject*)(&PyUnicode_Type))); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_data, __pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "binpt.pyx":14 + * return data + * elif isinstance(data, unicode): + * return data.encode('UTF-8') # <<<<<<<<<<<<<< + * raise TypeError('Cannot convert %s to string' % type(data)) + * + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + __pyx_t_1 = PyObject_GetAttr(__pyx_v_data, __pyx_n_s__encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_3)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_r = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + goto __pyx_L0; + goto __pyx_L3; + } + __pyx_L3:; + + /* "binpt.pyx":15 + * elif isinstance(data, unicode): + * return data.encode('UTF-8') + * raise TypeError('Cannot convert %s to string' % type(data)) # <<<<<<<<<<<<<< + * + * cdef class QueryResult(object): + */ + __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_3), ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_3)); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_t_3)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + + __pyx_r = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("binpt.as_str", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static int __pyx_pw_5binpt_11QueryResult_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5binpt_11QueryResult_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_words = 0; + PyObject *__pyx_v_scores = 0; + PyObject *__pyx_v_wa = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__words,&__pyx_n_s__scores,&__pyx_n_s__wa,0}; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + PyObject* values[3] = {0,0,0}; + + /* "binpt.pyx":29 + * cdef bytes _wa + * + * def __cinit__(self, words, scores, wa = None): # <<<<<<<<<<<<<< + * '''Requires a tuple of words (as strings) and a tuple of scores (as floats). + * Word-alignment info (as string) may be provided''' + */ + values[2] = ((PyObject *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__words); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__scores); + if (likely(values[1])) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__wa); + if (value) { values[2] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_words = values[0]; + __pyx_v_scores = values[1]; + __pyx_v_wa = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("binpt.QueryResult.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5binpt_11QueryResult___cinit__(((struct __pyx_obj_5binpt_QueryResult *)__pyx_v_self), __pyx_v_words, __pyx_v_scores, __pyx_v_wa); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5binpt_11QueryResult___cinit__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self, PyObject *__pyx_v_words, PyObject *__pyx_v_scores, PyObject *__pyx_v_wa) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "binpt.pyx":32 + * '''Requires a tuple of words (as strings) and a tuple of scores (as floats). + * Word-alignment info (as string) may be provided''' + * self._words = words # <<<<<<<<<<<<<< + * self._scores = scores + * self._wa = wa + */ + if (!(likely(PyTuple_CheckExact(__pyx_v_words))||((__pyx_v_words) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected tuple, got %.200s", Py_TYPE(__pyx_v_words)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_v_words); + __Pyx_GIVEREF(__pyx_v_words); + __Pyx_GOTREF(__pyx_v_self->_words); + __Pyx_DECREF(((PyObject *)__pyx_v_self->_words)); + __pyx_v_self->_words = ((PyObject*)__pyx_v_words); + + /* "binpt.pyx":33 + * Word-alignment info (as string) may be provided''' + * self._words = words + * self._scores = scores # <<<<<<<<<<<<<< + * self._wa = wa + * + */ + if (!(likely(PyTuple_CheckExact(__pyx_v_scores))||((__pyx_v_scores) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected tuple, got %.200s", Py_TYPE(__pyx_v_scores)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_v_scores); + __Pyx_GIVEREF(__pyx_v_scores); + __Pyx_GOTREF(__pyx_v_self->_scores); + __Pyx_DECREF(((PyObject *)__pyx_v_self->_scores)); + __pyx_v_self->_scores = ((PyObject*)__pyx_v_scores); + + /* "binpt.pyx":34 + * self._words = words + * self._scores = scores + * self._wa = wa # <<<<<<<<<<<<<< + * + * @property + */ + if (!(likely(PyBytes_CheckExact(__pyx_v_wa))||((__pyx_v_wa) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_wa)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_v_wa); + __Pyx_GIVEREF(__pyx_v_wa); + __Pyx_GOTREF(__pyx_v_self->_wa); + __Pyx_DECREF(((PyObject *)__pyx_v_self->_wa)); + __pyx_v_self->_wa = ((PyObject*)__pyx_v_wa); + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("binpt.QueryResult.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_3words(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static char __pyx_doc_5binpt_11QueryResult_2words[] = "Tuple of words (as strings)"; +static PyObject *__pyx_pw_5binpt_11QueryResult_3words(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("words (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_11QueryResult_2words(((struct __pyx_obj_5binpt_QueryResult *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":37 + * + * @property + * def words(self): # <<<<<<<<<<<<<< + * '''Tuple of words (as strings)''' + * return self._words + */ + +static PyObject *__pyx_pf_5binpt_11QueryResult_2words(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("words", 0); + + /* "binpt.pyx":39 + * def words(self): + * '''Tuple of words (as strings)''' + * return self._words # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_words)); + __pyx_r = ((PyObject *)__pyx_v_self->_words); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_5scores(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static char __pyx_doc_5binpt_11QueryResult_4scores[] = "Tuple of scores (as floats)"; +static PyObject *__pyx_pw_5binpt_11QueryResult_5scores(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("scores (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_11QueryResult_4scores(((struct __pyx_obj_5binpt_QueryResult *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":42 + * + * @property + * def scores(self): # <<<<<<<<<<<<<< + * '''Tuple of scores (as floats)''' + * return self._scores + */ + +static PyObject *__pyx_pf_5binpt_11QueryResult_4scores(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("scores", 0); + + /* "binpt.pyx":44 + * def scores(self): + * '''Tuple of scores (as floats)''' + * return self._scores # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_scores)); + __pyx_r = ((PyObject *)__pyx_v_self->_scores); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_7wa(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static char __pyx_doc_5binpt_11QueryResult_6wa[] = "Word-alignment info (as string)"; +static PyObject *__pyx_pw_5binpt_11QueryResult_7wa(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("wa (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_11QueryResult_6wa(((struct __pyx_obj_5binpt_QueryResult *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":47 + * + * @property + * def wa(self): # <<<<<<<<<<<<<< + * '''Word-alignment info (as string)''' + * return self._wa + */ + +static PyObject *__pyx_pf_5binpt_11QueryResult_6wa(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("wa", 0); + + /* "binpt.pyx":49 + * def wa(self): + * '''Word-alignment info (as string)''' + * return self._wa # <<<<<<<<<<<<<< + * + * @staticmethod + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_wa)); + __pyx_r = ((PyObject *)__pyx_v_self->_wa); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_9desc(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5binpt_11QueryResult_8desc[] = "Returns the sign of keys(y) - keys(x).\n Can only be used if scores is not an empty vector as\n keys defaults to scores[0]"; +static PyMethodDef __pyx_mdef_5binpt_11QueryResult_9desc = {__Pyx_NAMESTR("desc"), (PyCFunction)__pyx_pw_5binpt_11QueryResult_9desc, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(__pyx_doc_5binpt_11QueryResult_8desc)}; +static PyObject *__pyx_pw_5binpt_11QueryResult_9desc(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_x = 0; + PyObject *__pyx_v_y = 0; + PyObject *__pyx_v_keys = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__x,&__pyx_n_s__y,&__pyx_n_s__keys,0}; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("desc (wrapper)", 0); + { + PyObject* values[3] = {0,0,0}; + values[2] = __pyx_k_4; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__x); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__y); + if (likely(values[1])) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("desc", 0, 2, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__keys); + if (value) { values[2] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "desc") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_x = values[0]; + __pyx_v_y = values[1]; + __pyx_v_keys = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("desc", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("binpt.QueryResult.desc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5binpt_11QueryResult_8desc(__pyx_v_x, __pyx_v_y, __pyx_v_keys); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_4desc_lambda1(PyObject *__pyx_self, PyObject *__pyx_v_r); /*proto*/ +static PyMethodDef __pyx_mdef_5binpt_11QueryResult_4desc_lambda1 = {__Pyx_NAMESTR("lambda1"), (PyCFunction)__pyx_pw_5binpt_11QueryResult_4desc_lambda1, METH_O, __Pyx_DOCSTR(0)}; +static PyObject *__pyx_pw_5binpt_11QueryResult_4desc_lambda1(PyObject *__pyx_self, PyObject *__pyx_v_r) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda1 (wrapper)", 0); + __pyx_self = __pyx_self; + __pyx_r = __pyx_lambda_funcdef_lambda1(__pyx_self, ((PyObject *)__pyx_v_r)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":52 + * + * @staticmethod + * def desc(x, y, keys = lambda r: r.scores[0]): # <<<<<<<<<<<<<< + * '''Returns the sign of keys(y) - keys(x). + * Can only be used if scores is not an empty vector as + */ + +static PyObject *__pyx_lambda_funcdef_lambda1(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_r) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("lambda1", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyObject_GetAttr(__pyx_v_r, __pyx_n_s__scores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_1, 0, sizeof(long), PyInt_FromLong); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("binpt.QueryResult.desc.lambda1", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5binpt_11QueryResult_8desc(PyObject *__pyx_v_x, PyObject *__pyx_v_y, PyObject *__pyx_v_keys) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + float __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("desc", 0); + + /* "binpt.pyx":56 + * Can only be used if scores is not an empty vector as + * keys defaults to scores[0]''' + * return fsign(keys(y) - keys(x)) # <<<<<<<<<<<<<< + * + * def __str__(self): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_y); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_y); + __Pyx_GIVEREF(__pyx_v_y); + __pyx_t_2 = PyObject_Call(__pyx_v_keys, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_x); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_x); + __Pyx_GIVEREF(__pyx_v_x); + __pyx_t_3 = PyObject_Call(__pyx_v_keys, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_t_1 = PyNumber_Subtract(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_4 = __pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_4 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyInt_FromLong(__pyx_f_5binpt_fsign(__pyx_t_4, 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("binpt.QueryResult.desc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_11__str__(PyObject *__pyx_v_self); /*proto*/ +static char __pyx_doc_5binpt_11QueryResult_10__str__[] = "Returns a string such as: ||| [||| word-alignment info]"; +struct wrapperbase __pyx_wrapperbase_5binpt_11QueryResult_10__str__; +static PyObject *__pyx_pw_5binpt_11QueryResult_11__str__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_11QueryResult_10__str__(((struct __pyx_obj_5binpt_QueryResult *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":58 + * return fsign(keys(y) - keys(x)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * '''Returns a string such as: ||| [||| word-alignment info]''' + * if self._wa: + */ + +static PyObject *__pyx_pf_5binpt_11QueryResult_10__str__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self) { + PyObject *__pyx_v_x = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__str__", 0); + + /* "binpt.pyx":60 + * def __str__(self): + * '''Returns a string such as: ||| [||| word-alignment info]''' + * if self._wa: # <<<<<<<<<<<<<< + * return ' ||| '.join( (' '.join(self._words), + * ' '.join([str(x) for x in self._scores]), + */ + __pyx_t_1 = (((PyObject *)__pyx_v_self->_wa) != Py_None) && (PyBytes_GET_SIZE(((PyObject *)__pyx_v_self->_wa)) != 0); + if (__pyx_t_1) { + + /* "binpt.pyx":61 + * '''Returns a string such as: ||| [||| word-alignment info]''' + * if self._wa: + * return ' ||| '.join( (' '.join(self._words), # <<<<<<<<<<<<<< + * ' '.join([str(x) for x in self._scores]), + * self._wa) ) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_5), __pyx_n_s__join); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_words)); + PyTuple_SET_ITEM(__pyx_t_4, 0, ((PyObject *)__pyx_v_self->_words)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self->_words)); + __pyx_t_5 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + + /* "binpt.pyx":62 + * if self._wa: + * return ' ||| '.join( (' '.join(self._words), + * ' '.join([str(x) for x in self._scores]), # <<<<<<<<<<<<<< + * self._wa) ) + * else: + */ + __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(((PyObject *)__pyx_v_self->_scores) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_t_6 = ((PyObject *)__pyx_v_self->_scores); __Pyx_INCREF(__pyx_t_6); __pyx_t_7 = 0; + for (;;) { + if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_6)) break; + __pyx_t_8 = PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_7); __Pyx_INCREF(__pyx_t_8); __pyx_t_7++; + __Pyx_XDECREF(__pyx_v_x); + __pyx_v_x = __pyx_t_8; + __pyx_t_8 = 0; + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_v_x); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_x); + __Pyx_GIVEREF(__pyx_v_x); + __pyx_t_9 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + if (unlikely(PyList_Append(__pyx_t_3, (PyObject*)__pyx_t_9))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(((PyObject *)__pyx_t_3)); + PyTuple_SET_ITEM(__pyx_t_6, 0, ((PyObject *)__pyx_t_3)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + + /* "binpt.pyx":63 + * return ' ||| '.join( (' '.join(self._words), + * ' '.join([str(x) for x in self._scores]), + * self._wa) ) # <<<<<<<<<<<<<< + * else: + * return ' ||| '.join( (' '.join(self._words), + */ + __pyx_t_6 = PyTuple_New(3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_wa)); + PyTuple_SET_ITEM(__pyx_t_6, 2, ((PyObject *)__pyx_v_self->_wa)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self->_wa)); + __pyx_t_5 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_6)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_6)); + __pyx_t_6 = 0; + __pyx_t_6 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + goto __pyx_L3; + } + /*else*/ { + + /* "binpt.pyx":65 + * self._wa) ) + * else: + * return ' ||| '.join( (' '.join(self._words), # <<<<<<<<<<<<<< + * ' '.join([str(x) for x in self._scores]) ) ) + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_5), __pyx_n_s__join); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_words)); + PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_v_self->_words)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self->_words)); + __pyx_t_5 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + + /* "binpt.pyx":66 + * else: + * return ' ||| '.join( (' '.join(self._words), + * ' '.join([str(x) for x in self._scores]) ) ) # <<<<<<<<<<<<<< + * + * def __repr__(self): + */ + __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__join); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(((PyObject *)__pyx_v_self->_scores) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_t_4 = ((PyObject *)__pyx_v_self->_scores); __Pyx_INCREF(__pyx_t_4); __pyx_t_7 = 0; + for (;;) { + if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_4)) break; + __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; + __Pyx_XDECREF(__pyx_v_x); + __pyx_v_x = __pyx_t_9; + __pyx_t_9 = 0; + __pyx_t_9 = PyTuple_New(1); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_INCREF(__pyx_v_x); + PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_v_x); + __Pyx_GIVEREF(__pyx_v_x); + __pyx_t_8 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_9), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(((PyObject *)__pyx_t_9)); __pyx_t_9 = 0; + if (unlikely(PyList_Append(__pyx_t_3, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(((PyObject *)__pyx_t_3)); + PyTuple_SET_ITEM(__pyx_t_4, 0, ((PyObject *)__pyx_t_3)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_5 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_4)); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + } + __pyx_L3:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("binpt.QueryResult.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_x); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_11QueryResult_13__repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5binpt_11QueryResult_13__repr__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_11QueryResult_12__repr__(((struct __pyx_obj_5binpt_QueryResult *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":68 + * ' '.join([str(x) for x in self._scores]) ) ) + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return repr((repr(self._words), repr(self._scores), repr(self._wa))) + * + */ + +static PyObject *__pyx_pf_5binpt_11QueryResult_12__repr__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "binpt.pyx":69 + * + * def __repr__(self): + * return repr((repr(self._words), repr(self._scores), repr(self._wa))) # <<<<<<<<<<<<<< + * + * cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((PyObject *)__pyx_v_self->_words); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = ((PyObject *)__pyx_v_self->_scores); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_3 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = ((PyObject *)__pyx_v_self->_wa); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = PyObject_Repr(__pyx_t_1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_2 = 0; + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Repr(((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("binpt.QueryResult.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":71 + * return repr((repr(self._words), repr(self._scores), repr(self._wa))) + * + * cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None): # <<<<<<<<<<<<<< + * '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string) + * to a QueryResult (python object).''' + */ + +static struct __pyx_obj_5binpt_QueryResult *__pyx_f_5binpt_get_query_result(Moses::StringTgtCand &__pyx_v_cand, struct __pyx_opt_args_5binpt_get_query_result *__pyx_optional_args) { + PyObject *__pyx_v_wa = ((PyObject *)Py_None); + PyObject *__pyx_v_words = 0; + PyObject *__pyx_v_scores = 0; + size_t __pyx_v_i; + struct __pyx_obj_5binpt_QueryResult *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + size_t __pyx_t_2; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_query_result", 0); + if (__pyx_optional_args) { + if (__pyx_optional_args->__pyx_n > 0) { + __pyx_v_wa = __pyx_optional_args->wa; + } + } + + /* "binpt.pyx":74 + * '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string) + * to a QueryResult (python object).''' + * cdef tuple words = tuple([cand.first[i].c_str() for i in range(cand.first.size())]) # <<<<<<<<<<<<<< + * cdef tuple scores = tuple([cand.second[i] for i in range(cand.second.size())]) + * return QueryResult(words, scores, wa) + */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_v_cand.first.size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + __pyx_t_4 = PyBytes_FromString((__pyx_v_cand.first[__pyx_v_i])->c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + if (unlikely(PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + } + __pyx_t_4 = ((PyObject *)PyList_AsTuple(__pyx_t_1)); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_v_words = __pyx_t_4; + __pyx_t_4 = 0; + + /* "binpt.pyx":75 + * to a QueryResult (python object).''' + * cdef tuple words = tuple([cand.first[i].c_str() for i in range(cand.first.size())]) + * cdef tuple scores = tuple([cand.second[i] for i in range(cand.second.size())]) # <<<<<<<<<<<<<< + * return QueryResult(words, scores, wa) + * + */ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __pyx_v_cand.second.size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + __pyx_t_1 = PyFloat_FromDouble((__pyx_v_cand.second[__pyx_v_i])); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (unlikely(PyList_Append(__pyx_t_4, (PyObject*)__pyx_t_1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_t_1 = ((PyObject *)PyList_AsTuple(__pyx_t_4)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __pyx_v_scores = __pyx_t_1; + __pyx_t_1 = 0; + + /* "binpt.pyx":76 + * cdef tuple words = tuple([cand.first[i].c_str() for i in range(cand.first.size())]) + * cdef tuple scores = tuple([cand.second[i] for i in range(cand.second.size())]) + * return QueryResult(words, scores, wa) # <<<<<<<<<<<<<< + * + * cdef class BinaryPhraseTable(object): + */ + __Pyx_XDECREF(((PyObject *)__pyx_r)); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)__pyx_v_words)); + PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_words)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_words)); + __Pyx_INCREF(((PyObject *)__pyx_v_scores)); + PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_scores)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_scores)); + __Pyx_INCREF(__pyx_v_wa); + PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_wa); + __Pyx_GIVEREF(__pyx_v_wa); + __pyx_t_4 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5binpt_QueryResult)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_r = ((struct __pyx_obj_5binpt_QueryResult *)__pyx_t_4); + __pyx_t_4 = 0; + goto __pyx_L0; + + __pyx_r = ((struct __pyx_obj_5binpt_QueryResult *)Py_None); __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("binpt.get_query_result", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_words); + __Pyx_XDECREF(__pyx_v_scores); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_path = 0; + unsigned int __pyx_v_nscores; + int __pyx_v_wa; + PyObject *__pyx_v_delimiters = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__path,&__pyx_n_s__nscores,&__pyx_n_s__wa,&__pyx_n_s__delimiters,0}; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + PyObject* values[4] = {0,0,0,0}; + values[3] = ((PyObject *)__pyx_kp_s_7); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__path); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__nscores); + if (value) { values[1] = value; kw_args--; } + } + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__wa); + if (value) { values[2] = value; kw_args--; } + } + case 3: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__delimiters); + if (value) { values[3] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + if (values[1]) { + } else { + __pyx_v_nscores = ((unsigned int)5); + } + if (values[2]) { + } else { + + /* "binpt.pyx":88 + * cdef bytes _delimiters + * + * def __cinit__(self, bytes path, unsigned nscores = 5, bint wa = False, delimiters = ' \t'): # <<<<<<<<<<<<<< + * '''It requies a path to binary phrase table (stem of the table, e.g europarl.fr-en + * is the stem for europar.fr-en.binphr.*). + */ + __pyx_v_wa = ((int)0); + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_path = ((PyObject*)values[0]); + if (values[1]) { + __pyx_v_nscores = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_nscores == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_nscores = ((unsigned int)5); + } + if (values[2]) { + __pyx_v_wa = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_wa == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_wa = ((int)0); + } + __pyx_v_delimiters = values[3]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 1, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("binpt.BinaryPhraseTable.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_path), (&PyBytes_Type), 1, "path", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self), __pyx_v_path, __pyx_v_nscores, __pyx_v_wa, __pyx_v_delimiters); + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_path, unsigned int __pyx_v_nscores, int __pyx_v_wa, PyObject *__pyx_v_delimiters) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + char *__pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "binpt.pyx":95 + * One can also specify the token delimiters, for Moses::Tokenize(text, delimiters), which is space or tab by default.''' + * + * if not BinaryPhraseTable.isValidBinaryTable(path, wa): # <<<<<<<<<<<<<< + * raise ValueError, "'%s' doesn't seem a valid binary table." % path + * self._path = path + */ + __pyx_t_1 = PyObject_GetAttr(((PyObject *)((PyObject*)__pyx_ptype_5binpt_BinaryPhraseTable)), __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_wa); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(((PyObject *)__pyx_v_path)); + PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_v_path)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_path)); + PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (!__pyx_t_4); + if (__pyx_t_5) { + + /* "binpt.pyx":96 + * + * if not BinaryPhraseTable.isValidBinaryTable(path, wa): + * raise ValueError, "'%s' doesn't seem a valid binary table." % path # <<<<<<<<<<<<<< + * self._path = path + * self._nscores = nscores + */ + __pyx_t_2 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_8), ((PyObject *)__pyx_v_path)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_Raise(__pyx_builtin_ValueError, ((PyObject *)__pyx_t_2), 0, 0); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + goto __pyx_L3; + } + __pyx_L3:; + + /* "binpt.pyx":97 + * if not BinaryPhraseTable.isValidBinaryTable(path, wa): + * raise ValueError, "'%s' doesn't seem a valid binary table." % path + * self._path = path # <<<<<<<<<<<<<< + * self._nscores = nscores + * self._wa = wa + */ + __Pyx_INCREF(((PyObject *)__pyx_v_path)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_path)); + __Pyx_GOTREF(__pyx_v_self->_path); + __Pyx_DECREF(((PyObject *)__pyx_v_self->_path)); + __pyx_v_self->_path = __pyx_v_path; + + /* "binpt.pyx":98 + * raise ValueError, "'%s' doesn't seem a valid binary table." % path + * self._path = path + * self._nscores = nscores # <<<<<<<<<<<<<< + * self._wa = wa + * self._delimiters = delimiters + */ + __pyx_v_self->_nscores = __pyx_v_nscores; + + /* "binpt.pyx":99 + * self._path = path + * self._nscores = nscores + * self._wa = wa # <<<<<<<<<<<<<< + * self._delimiters = delimiters + * self.__tree = new PhraseDictionaryTree(nscores) + */ + __pyx_v_self->_wa = __pyx_v_wa; + + /* "binpt.pyx":100 + * self._nscores = nscores + * self._wa = wa + * self._delimiters = delimiters # <<<<<<<<<<<<<< + * self.__tree = new PhraseDictionaryTree(nscores) + * self.__tree.UseWordAlignment(wa) + */ + if (!(likely(PyBytes_CheckExact(__pyx_v_delimiters))||((__pyx_v_delimiters) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_delimiters)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_v_delimiters); + __Pyx_GIVEREF(__pyx_v_delimiters); + __Pyx_GOTREF(__pyx_v_self->_delimiters); + __Pyx_DECREF(((PyObject *)__pyx_v_self->_delimiters)); + __pyx_v_self->_delimiters = ((PyObject*)__pyx_v_delimiters); + + /* "binpt.pyx":101 + * self._wa = wa + * self._delimiters = delimiters + * self.__tree = new PhraseDictionaryTree(nscores) # <<<<<<<<<<<<<< + * self.__tree.UseWordAlignment(wa) + * self.__tree.Read(string(path)) + */ + __pyx_v_self->__pyx___tree = new Moses::PhraseDictionaryTree(__pyx_v_nscores); + + /* "binpt.pyx":102 + * self._delimiters = delimiters + * self.__tree = new PhraseDictionaryTree(nscores) + * self.__tree.UseWordAlignment(wa) # <<<<<<<<<<<<<< + * self.__tree.Read(string(path)) + * + */ + __pyx_v_self->__pyx___tree->UseWordAlignment(__pyx_v_wa); + + /* "binpt.pyx":103 + * self.__tree = new PhraseDictionaryTree(nscores) + * self.__tree.UseWordAlignment(wa) + * self.__tree.Read(string(path)) # <<<<<<<<<<<<<< + * + * def __dealloc__(self): + */ + __pyx_t_6 = PyBytes_AsString(((PyObject *)__pyx_v_path)); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_self->__pyx___tree->Read(std::string(__pyx_t_6)); + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("binpt.BinaryPhraseTable.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static void __pyx_pw_5binpt_17BinaryPhraseTable_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5binpt_17BinaryPhraseTable_3__dealloc__(PyObject *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_pf_5binpt_17BinaryPhraseTable_2__dealloc__(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); +} + +/* "binpt.pyx":105 + * self.__tree.Read(string(path)) + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self.__tree + * + */ + +static void __pyx_pf_5binpt_17BinaryPhraseTable_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__", 0); + + /* "binpt.pyx":106 + * + * def __dealloc__(self): + * del self.__tree # <<<<<<<<<<<<<< + * + * @staticmethod + */ + delete __pyx_v_self->__pyx___tree; + + __Pyx_RefNannyFinishContext(); +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_5isValidBinaryTable(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5binpt_17BinaryPhraseTable_4isValidBinaryTable[] = "This sanity check was added to the constructor, but you can access it from outside this class\n to determine whether or not you are providing a valid stem to BinaryPhraseTable."; +static PyMethodDef __pyx_mdef_5binpt_17BinaryPhraseTable_5isValidBinaryTable = {__Pyx_NAMESTR("isValidBinaryTable"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_5isValidBinaryTable, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(__pyx_doc_5binpt_17BinaryPhraseTable_4isValidBinaryTable)}; +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_5isValidBinaryTable(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_stem = 0; + int __pyx_v_wa; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__stem,&__pyx_n_s__wa,0}; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("isValidBinaryTable (wrapper)", 0); + { + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__stem); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__wa); + if (value) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "isValidBinaryTable") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + if (values[1]) { + } else { + + /* "binpt.pyx":109 + * + * @staticmethod + * def isValidBinaryTable(stem, bint wa = False): # <<<<<<<<<<<<<< + * '''This sanity check was added to the constructor, but you can access it from outside this class + * to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + */ + __pyx_v_wa = ((int)0); + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_stem = values[0]; + if (values[1]) { + __pyx_v_wa = __Pyx_PyObject_IsTrue(values[1]); if (unlikely((__pyx_v_wa == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_wa = ((int)0); + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("isValidBinaryTable", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("binpt.BinaryPhraseTable.isValidBinaryTable", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(__pyx_v_stem, __pyx_v_wa); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObject *__pyx_v_stem, int __pyx_v_wa) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("isValidBinaryTable", 0); + + /* "binpt.pyx":112 + * '''This sanity check was added to the constructor, but you can access it from outside this class + * to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + * if wa: # <<<<<<<<<<<<<< + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree.wa") \ + */ + if (__pyx_v_wa) { + + /* "binpt.pyx":113 + * to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + * if wa: + * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srctree.wa") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ + */ + __Pyx_XDECREF(__pyx_r); + + /* "binpt.pyx":114 + * if wa: + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree.wa") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ + */ + __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__isfile); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "binpt.pyx":113 + * to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + * if wa: + * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srctree.wa") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ + */ + __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "binpt.pyx":114 + * if wa: + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree.wa") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ + */ + __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "binpt.pyx":115 + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree.wa") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ + * and os.path.isfile(stem + ".binphr.tgtvoc") + */ + __pyx_t_5 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_5, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "binpt.pyx":116 + * and os.path.isfile(stem + ".binphr.srctree.wa") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.tgtvoc") + * else: + */ + __pyx_t_6 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_12)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_5); + __pyx_t_5 = 0; + __pyx_t_5 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "binpt.pyx":117 + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata.wa") \ + * and os.path.isfile(stem + ".binphr.tgtvoc") # <<<<<<<<<<<<<< + * else: + * return os.path.isfile(stem + ".binphr.idx") \ + */ + __pyx_t_7 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = PyObject_GetAttr(__pyx_t_7, __pyx_n_s__path); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__isfile); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); + __Pyx_GIVEREF(__pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __pyx_t_8 = __pyx_t_6; + __pyx_t_6 = 0; + } else { + __pyx_t_8 = __pyx_t_5; + __pyx_t_5 = 0; + } + __pyx_t_5 = __pyx_t_8; + __pyx_t_8 = 0; + } else { + __pyx_t_5 = __pyx_t_3; + __pyx_t_3 = 0; + } + __pyx_t_3 = __pyx_t_5; + __pyx_t_5 = 0; + } else { + __pyx_t_3 = __pyx_t_1; + __pyx_t_1 = 0; + } + __pyx_t_1 = __pyx_t_3; + __pyx_t_3 = 0; + } else { + __pyx_t_1 = __pyx_t_2; + __pyx_t_2 = 0; + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + goto __pyx_L3; + } + /*else*/ { + + /* "binpt.pyx":119 + * and os.path.isfile(stem + ".binphr.tgtvoc") + * else: + * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srctree") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ + */ + __Pyx_XDECREF(__pyx_r); + + /* "binpt.pyx":120 + * else: + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata") \ + */ + __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__isfile); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "binpt.pyx":119 + * and os.path.isfile(stem + ".binphr.tgtvoc") + * else: + * return os.path.isfile(stem + ".binphr.idx") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srctree") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ + */ + __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "binpt.pyx":120 + * else: + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata") \ + */ + __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_14)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 120; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "binpt.pyx":121 + * return os.path.isfile(stem + ".binphr.idx") \ + * and os.path.isfile(stem + ".binphr.srctree") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.tgtdata") \ + * and os.path.isfile(stem + ".binphr.tgtvoc") + */ + __pyx_t_5 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_5, ((PyObject *)__pyx_t_8), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "binpt.pyx":122 + * and os.path.isfile(stem + ".binphr.srctree") \ + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata") \ # <<<<<<<<<<<<<< + * and os.path.isfile(stem + ".binphr.tgtvoc") + * + */ + __pyx_t_8 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_15)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_5); + __pyx_t_5 = 0; + __pyx_t_5 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "binpt.pyx":123 + * and os.path.isfile(stem + ".binphr.srcvoc") \ + * and os.path.isfile(stem + ".binphr.tgtdata") \ + * and os.path.isfile(stem + ".binphr.tgtvoc") # <<<<<<<<<<<<<< + * + * @property + */ + __pyx_t_6 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__path); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); + __Pyx_GIVEREF(__pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + __pyx_t_7 = __pyx_t_8; + __pyx_t_8 = 0; + } else { + __pyx_t_7 = __pyx_t_5; + __pyx_t_5 = 0; + } + __pyx_t_5 = __pyx_t_7; + __pyx_t_7 = 0; + } else { + __pyx_t_5 = __pyx_t_3; + __pyx_t_3 = 0; + } + __pyx_t_3 = __pyx_t_5; + __pyx_t_5 = 0; + } else { + __pyx_t_3 = __pyx_t_1; + __pyx_t_1 = 0; + } + __pyx_t_1 = __pyx_t_3; + __pyx_t_3 = 0; + } else { + __pyx_t_1 = __pyx_t_2; + __pyx_t_2 = 0; + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + } + __pyx_L3:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("binpt.BinaryPhraseTable.isValidBinaryTable", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_7path(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_7path(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("path (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_6path(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":126 + * + * @property + * def path(self): # <<<<<<<<<<<<<< + * return self._path + * + */ + +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_6path(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("path", 0); + + /* "binpt.pyx":127 + * @property + * def path(self): + * return self._path # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_path)); + __pyx_r = ((PyObject *)__pyx_v_self->_path); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_9nscores(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_9nscores(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("nscores (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_8nscores(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":130 + * + * @property + * def nscores(self): # <<<<<<<<<<<<<< + * return self._nscores + * + */ + +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_8nscores(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("nscores", 0); + + /* "binpt.pyx":131 + * @property + * def nscores(self): + * return self._nscores # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_self->_nscores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("binpt.BinaryPhraseTable.nscores", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_11wa(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_11wa(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("wa (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_10wa(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":134 + * + * @property + * def wa(self): # <<<<<<<<<<<<<< + * return self._wa + * + */ + +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_10wa(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("wa", 0); + + /* "binpt.pyx":135 + * @property + * def wa(self): + * return self._wa # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->_wa); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("binpt.BinaryPhraseTable.wa", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiters(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiters(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("delimiters (wrapper)", 0); + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_12delimiters(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "binpt.pyx":138 + * + * @property + * def delimiters(self): # <<<<<<<<<<<<<< + * return self._delimiters + * + */ + +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiters(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("delimiters", 0); + + /* "binpt.pyx":139 + * @property + * def delimiters(self): + * return self._delimiters # <<<<<<<<<<<<<< + * + * def query(self, line, cmp = None, top = 0): + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self->_delimiters)); + __pyx_r = ((PyObject *)__pyx_v_self->_delimiters); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_15query(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5binpt_17BinaryPhraseTable_14query[] = "Queries the phrase table and returns a list of matches.\n Each match is a QueryResult.\n If 'cmp' is defined the return list is sorted.\n If 'top' is defined, onlye the top elements will be returned."; +static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_15query(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_line = 0; + PyObject *__pyx_v_cmp = 0; + PyObject *__pyx_v_top = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__line,&__pyx_n_s__cmp,&__pyx_n_s__top,0}; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("query (wrapper)", 0); + { + PyObject* values[3] = {0,0,0}; + + /* "binpt.pyx":141 + * return self._delimiters + * + * def query(self, line, cmp = None, top = 0): # <<<<<<<<<<<<<< + * '''Queries the phrase table and returns a list of matches. + * Each match is a QueryResult. + */ + values[1] = ((PyObject *)Py_None); + values[2] = ((PyObject *)__pyx_int_0); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__line); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__cmp); + if (value) { values[1] = value; kw_args--; } + } + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__top); + if (value) { values[2] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "query") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_line = values[0]; + __pyx_v_cmp = values[1]; + __pyx_v_top = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("query", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("binpt.BinaryPhraseTable.query", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_14query(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self), __pyx_v_line, __pyx_v_cmp, __pyx_v_top); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_line, PyObject *__pyx_v_cmp, PyObject *__pyx_v_top) { + PyObject *__pyx_v_text = 0; + std::vector __pyx_v_fphrase; + std::vector *__pyx_v_rv; + std::vector *__pyx_v_wa; + PyObject *__pyx_v_phrases = 0; + size_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + char *__pyx_t_2; + char *__pyx_t_3; + int __pyx_t_4; + size_t __pyx_t_5; + size_t __pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + struct __pyx_opt_args_5binpt_get_query_result __pyx_t_9; + Py_ssize_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("query", 0); + + /* "binpt.pyx":146 + * If 'cmp' is defined the return list is sorted. + * If 'top' is defined, onlye the top elements will be returned.''' + * cdef bytes text = as_str(line) # <<<<<<<<<<<<<< + * cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters)) + * cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]() + */ + __pyx_t_1 = ((PyObject *)__pyx_f_5binpt_as_str(__pyx_v_line)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_text = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "binpt.pyx":147 + * If 'top' is defined, onlye the top elements will be returned.''' + * cdef bytes text = as_str(line) + * cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters)) # <<<<<<<<<<<<<< + * cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]() + * cdef vector[string]* wa = NULL + */ + __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_text)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyBytes_AsString(((PyObject *)__pyx_v_self->_delimiters)); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_fphrase = Moses::Tokenize(std::string(__pyx_t_2), std::string(__pyx_t_3)); + + /* "binpt.pyx":148 + * cdef bytes text = as_str(line) + * cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters)) + * cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]() # <<<<<<<<<<<<<< + * cdef vector[string]* wa = NULL + * cdef list phrases + */ + __pyx_v_rv = new std::vector(); + + /* "binpt.pyx":149 + * cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters)) + * cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]() + * cdef vector[string]* wa = NULL # <<<<<<<<<<<<<< + * cdef list phrases + * if not self.__tree.UseWordAlignment(): + */ + __pyx_v_wa = NULL; + + /* "binpt.pyx":151 + * cdef vector[string]* wa = NULL + * cdef list phrases + * if not self.__tree.UseWordAlignment(): # <<<<<<<<<<<<<< + * self.__tree.GetTargetCandidates(fphrase, rv[0]) + * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] + */ + __pyx_t_4 = (!__pyx_v_self->__pyx___tree->UseWordAlignment()); + if (__pyx_t_4) { + + /* "binpt.pyx":152 + * cdef list phrases + * if not self.__tree.UseWordAlignment(): + * self.__tree.GetTargetCandidates(fphrase, rv[0]) # <<<<<<<<<<<<<< + * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] + * else: + */ + __pyx_v_self->__pyx___tree->GetTargetCandidates(__pyx_v_fphrase, (__pyx_v_rv[0])); + + /* "binpt.pyx":153 + * if not self.__tree.UseWordAlignment(): + * self.__tree.GetTargetCandidates(fphrase, rv[0]) + * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] # <<<<<<<<<<<<<< + * else: + * wa = new vector[string]() + */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = __pyx_v_rv->size(); + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + __pyx_t_7 = ((PyObject *)__pyx_f_5binpt_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), NULL)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_7))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } + __Pyx_INCREF(((PyObject *)__pyx_t_1)); + __pyx_v_phrases = __pyx_t_1; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + goto __pyx_L3; + } + /*else*/ { + + /* "binpt.pyx":155 + * phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] + * else: + * wa = new vector[string]() # <<<<<<<<<<<<<< + * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) + * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] + */ + __pyx_v_wa = new std::vector(); + + /* "binpt.pyx":156 + * else: + * wa = new vector[string]() + * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) # <<<<<<<<<<<<<< + * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] + * del wa + */ + __pyx_v_self->__pyx___tree->GetTargetCandidates(__pyx_v_fphrase, (__pyx_v_rv[0]), (__pyx_v_wa[0])); + + /* "binpt.pyx":157 + * wa = new vector[string]() + * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) + * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] # <<<<<<<<<<<<<< + * del wa + * del rv + */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = __pyx_v_rv->size(); + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + __pyx_t_7 = PyBytes_FromString(((__pyx_v_wa[0])[__pyx_v_i]).c_str()); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __pyx_t_9.__pyx_n = 1; + __pyx_t_9.wa = ((PyObject *)__pyx_t_7); + __pyx_t_8 = ((PyObject *)__pyx_f_5binpt_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), &__pyx_t_9)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + if (unlikely(PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_INCREF(((PyObject *)__pyx_t_1)); + __pyx_v_phrases = __pyx_t_1; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + + /* "binpt.pyx":158 + * self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) + * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] + * del wa # <<<<<<<<<<<<<< + * del rv + * if cmp: + */ + delete __pyx_v_wa; + } + __pyx_L3:; + + /* "binpt.pyx":159 + * phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] + * del wa + * del rv # <<<<<<<<<<<<<< + * if cmp: + * phrases.sort(cmp=cmp) + */ + delete __pyx_v_rv; + + /* "binpt.pyx":160 + * del wa + * del rv + * if cmp: # <<<<<<<<<<<<<< + * phrases.sort(cmp=cmp) + * if top > 0: + */ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_cmp); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_4) { + + /* "binpt.pyx":161 + * del rv + * if cmp: + * phrases.sort(cmp=cmp) # <<<<<<<<<<<<<< + * if top > 0: + * return phrases[0:top] + */ + __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_phrases), __pyx_n_s__sort); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_8 = PyDict_New(); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); + if (PyDict_SetItem(__pyx_t_8, ((PyObject *)__pyx_n_s__cmp), __pyx_v_cmp) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), ((PyObject *)__pyx_t_8)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L8; + } + __pyx_L8:; + + /* "binpt.pyx":162 + * if cmp: + * phrases.sort(cmp=cmp) + * if top > 0: # <<<<<<<<<<<<<< + * return phrases[0:top] + * else: + */ + __pyx_t_7 = PyObject_RichCompare(__pyx_v_top, __pyx_int_0, Py_GT); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 162; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_7); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 162; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_4) { + + /* "binpt.pyx":163 + * phrases.sort(cmp=cmp) + * if top > 0: + * return phrases[0:top] # <<<<<<<<<<<<<< + * else: + * return phrases + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_10 = __Pyx_PyIndex_AsSsize_t(__pyx_v_top); if (unlikely((__pyx_t_10 == (Py_ssize_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 163; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PySequence_GetSlice(((PyObject *)__pyx_v_phrases), 0, __pyx_t_10); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 163; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __pyx_r = ((PyObject *)__pyx_t_7); + __pyx_t_7 = 0; + goto __pyx_L0; + goto __pyx_L9; + } + /*else*/ { + + /* "binpt.pyx":165 + * return phrases[0:top] + * else: + * return phrases # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_phrases)); + __pyx_r = ((PyObject *)__pyx_v_phrases); + goto __pyx_L0; + } + __pyx_L9:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("binpt.BinaryPhraseTable.query", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_text); + __Pyx_XDECREF(__pyx_v_phrases); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_5binpt_QueryResult(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5binpt_QueryResult *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_5binpt_QueryResult *)o); + p->_words = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_scores = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_wa = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (__pyx_pw_5binpt_11QueryResult_1__cinit__(o, a, k) < 0) { + Py_DECREF(o); o = 0; + } + return o; +} + +static void __pyx_tp_dealloc_5binpt_QueryResult(PyObject *o) { + struct __pyx_obj_5binpt_QueryResult *p = (struct __pyx_obj_5binpt_QueryResult *)o; + Py_XDECREF(((PyObject *)p->_words)); + Py_XDECREF(((PyObject *)p->_scores)); + Py_XDECREF(((PyObject *)p->_wa)); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_5binpt_QueryResult(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5binpt_QueryResult *p = (struct __pyx_obj_5binpt_QueryResult *)o; + if (p->_words) { + e = (*v)(p->_words, a); if (e) return e; + } + if (p->_scores) { + e = (*v)(p->_scores, a); if (e) return e; + } + if (p->_wa) { + e = (*v)(p->_wa, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5binpt_QueryResult(PyObject *o) { + struct __pyx_obj_5binpt_QueryResult *p = (struct __pyx_obj_5binpt_QueryResult *)o; + PyObject* tmp; + tmp = ((PyObject*)p->_words); + p->_words = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_scores); + p->_scores = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_wa); + p->_wa = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_5binpt_QueryResult[] = { + {__Pyx_NAMESTR("words"), (PyCFunction)__pyx_pw_5binpt_11QueryResult_3words, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_5binpt_11QueryResult_2words)}, + {__Pyx_NAMESTR("scores"), (PyCFunction)__pyx_pw_5binpt_11QueryResult_5scores, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_5binpt_11QueryResult_4scores)}, + {__Pyx_NAMESTR("wa"), (PyCFunction)__pyx_pw_5binpt_11QueryResult_7wa, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_5binpt_11QueryResult_6wa)}, + {__Pyx_NAMESTR("desc"), (PyCFunction)__pyx_pw_5binpt_11QueryResult_9desc, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(__pyx_doc_5binpt_11QueryResult_8desc)}, + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_QueryResult = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_long*/ + #else + 0, /*reserved*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if PY_VERSION_HEX >= 0x02050000 + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_QueryResult = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_QueryResult = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_QueryResult = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +static PyTypeObject __pyx_type_5binpt_QueryResult = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("binpt.QueryResult"), /*tp_name*/ + sizeof(struct __pyx_obj_5binpt_QueryResult), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5binpt_QueryResult, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #else + 0, /*reserved*/ + #endif + __pyx_pw_5binpt_11QueryResult_13__repr__, /*tp_repr*/ + &__pyx_tp_as_number_QueryResult, /*tp_as_number*/ + &__pyx_tp_as_sequence_QueryResult, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_QueryResult, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + __pyx_pw_5binpt_11QueryResult_11__str__, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_QueryResult, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __Pyx_DOCSTR("This class represents a query result, that is,\n a target phrase (tuple of words/strings),\n a feature vector (tuple of floats)\n and possibly an alignment info (string).\n Here we don't bother parsing the alignment info, as it's often only\n used as is, threfore saving some time."), /*tp_doc*/ + __pyx_tp_traverse_5binpt_QueryResult, /*tp_traverse*/ + __pyx_tp_clear_5binpt_QueryResult, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5binpt_QueryResult, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5binpt_QueryResult, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + #if PY_VERSION_HEX >= 0x02060000 + 0, /*tp_version_tag*/ + #endif +}; + +static PyObject *__pyx_tp_new_5binpt_BinaryPhraseTable(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5binpt_BinaryPhraseTable *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_5binpt_BinaryPhraseTable *)o); + p->_path = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_delimiters = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (__pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(o, a, k) < 0) { + Py_DECREF(o); o = 0; + } + return o; +} + +static void __pyx_tp_dealloc_5binpt_BinaryPhraseTable(PyObject *o) { + struct __pyx_obj_5binpt_BinaryPhraseTable *p = (struct __pyx_obj_5binpt_BinaryPhraseTable *)o; + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + ++Py_REFCNT(o); + __pyx_pw_5binpt_17BinaryPhraseTable_3__dealloc__(o); + if (PyErr_Occurred()) PyErr_WriteUnraisable(o); + --Py_REFCNT(o); + PyErr_Restore(etype, eval, etb); + } + Py_XDECREF(((PyObject *)p->_path)); + Py_XDECREF(((PyObject *)p->_delimiters)); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_5binpt_BinaryPhraseTable(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5binpt_BinaryPhraseTable *p = (struct __pyx_obj_5binpt_BinaryPhraseTable *)o; + if (p->_path) { + e = (*v)(p->_path, a); if (e) return e; + } + if (p->_delimiters) { + e = (*v)(p->_delimiters, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5binpt_BinaryPhraseTable(PyObject *o) { + struct __pyx_obj_5binpt_BinaryPhraseTable *p = (struct __pyx_obj_5binpt_BinaryPhraseTable *)o; + PyObject* tmp; + tmp = ((PyObject*)p->_path); + p->_path = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_delimiters); + p->_delimiters = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_5binpt_BinaryPhraseTable[] = { + {__Pyx_NAMESTR("isValidBinaryTable"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_5isValidBinaryTable, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(__pyx_doc_5binpt_17BinaryPhraseTable_4isValidBinaryTable)}, + {__Pyx_NAMESTR("path"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_7path, METH_NOARGS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("nscores"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_9nscores, METH_NOARGS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("wa"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_11wa, METH_NOARGS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("delimiters"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_13delimiters, METH_NOARGS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("query"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_15query, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(__pyx_doc_5binpt_17BinaryPhraseTable_14query)}, + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_BinaryPhraseTable = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_long*/ + #else + 0, /*reserved*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if PY_VERSION_HEX >= 0x02050000 + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_BinaryPhraseTable = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_BinaryPhraseTable = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_BinaryPhraseTable = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +static PyTypeObject __pyx_type_5binpt_BinaryPhraseTable = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("binpt.BinaryPhraseTable"), /*tp_name*/ + sizeof(struct __pyx_obj_5binpt_BinaryPhraseTable), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5binpt_BinaryPhraseTable, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #else + 0, /*reserved*/ + #endif + 0, /*tp_repr*/ + &__pyx_tp_as_number_BinaryPhraseTable, /*tp_as_number*/ + &__pyx_tp_as_sequence_BinaryPhraseTable, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_BinaryPhraseTable, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_BinaryPhraseTable, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __Pyx_DOCSTR("This class encapsulates a Moses::PhraseDictionaryTree for operations over\n binary phrase tables."), /*tp_doc*/ + __pyx_tp_traverse_5binpt_BinaryPhraseTable, /*tp_traverse*/ + __pyx_tp_clear_5binpt_BinaryPhraseTable, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5binpt_BinaryPhraseTable, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5binpt_BinaryPhraseTable, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + #if PY_VERSION_HEX >= 0x02060000 + 0, /*tp_version_tag*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {__Pyx_NAMESTR("fsign"), (PyCFunction)__pyx_pw_5binpt_1fsign, METH_O, __Pyx_DOCSTR(__pyx_doc_5binpt_fsign)}, + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + PyModuleDef_HEAD_INIT, + __Pyx_NAMESTR("binpt"), + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp_s_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 0, 1, 0}, + {&__pyx_kp_s_10, __pyx_k_10, sizeof(__pyx_k_10), 0, 0, 1, 0}, + {&__pyx_kp_s_11, __pyx_k_11, sizeof(__pyx_k_11), 0, 0, 1, 0}, + {&__pyx_kp_s_12, __pyx_k_12, sizeof(__pyx_k_12), 0, 0, 1, 0}, + {&__pyx_kp_s_13, __pyx_k_13, sizeof(__pyx_k_13), 0, 0, 1, 0}, + {&__pyx_kp_s_14, __pyx_k_14, sizeof(__pyx_k_14), 0, 0, 1, 0}, + {&__pyx_kp_s_15, __pyx_k_15, sizeof(__pyx_k_15), 0, 0, 1, 0}, + {&__pyx_kp_s_18, __pyx_k_18, sizeof(__pyx_k_18), 0, 0, 1, 0}, + {&__pyx_kp_s_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 1, 0}, + {&__pyx_kp_s_5, __pyx_k_5, sizeof(__pyx_k_5), 0, 0, 1, 0}, + {&__pyx_kp_s_6, __pyx_k_6, sizeof(__pyx_k_6), 0, 0, 1, 0}, + {&__pyx_kp_s_7, __pyx_k_7, sizeof(__pyx_k_7), 0, 0, 1, 0}, + {&__pyx_kp_s_8, __pyx_k_8, sizeof(__pyx_k_8), 0, 0, 1, 0}, + {&__pyx_kp_s_9, __pyx_k_9, sizeof(__pyx_k_9), 0, 0, 1, 0}, + {&__pyx_n_s__TypeError, __pyx_k__TypeError, sizeof(__pyx_k__TypeError), 0, 0, 1, 1}, + {&__pyx_n_s__ValueError, __pyx_k__ValueError, sizeof(__pyx_k__ValueError), 0, 0, 1, 1}, + {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1}, + {&__pyx_n_s____test__, __pyx_k____test__, sizeof(__pyx_k____test__), 0, 0, 1, 1}, + {&__pyx_n_s__binpt, __pyx_k__binpt, sizeof(__pyx_k__binpt), 0, 0, 1, 1}, + {&__pyx_n_s__cmp, __pyx_k__cmp, sizeof(__pyx_k__cmp), 0, 0, 1, 1}, + {&__pyx_n_s__delimiters, __pyx_k__delimiters, sizeof(__pyx_k__delimiters), 0, 0, 1, 1}, + {&__pyx_n_s__desc, __pyx_k__desc, sizeof(__pyx_k__desc), 0, 0, 1, 1}, + {&__pyx_n_s__encode, __pyx_k__encode, sizeof(__pyx_k__encode), 0, 0, 1, 1}, + {&__pyx_n_s__isValidBinaryTable, __pyx_k__isValidBinaryTable, sizeof(__pyx_k__isValidBinaryTable), 0, 0, 1, 1}, + {&__pyx_n_s__isfile, __pyx_k__isfile, sizeof(__pyx_k__isfile), 0, 0, 1, 1}, + {&__pyx_n_s__join, __pyx_k__join, sizeof(__pyx_k__join), 0, 0, 1, 1}, + {&__pyx_n_s__keys, __pyx_k__keys, sizeof(__pyx_k__keys), 0, 0, 1, 1}, + {&__pyx_n_s__line, __pyx_k__line, sizeof(__pyx_k__line), 0, 0, 1, 1}, + {&__pyx_n_s__nscores, __pyx_k__nscores, sizeof(__pyx_k__nscores), 0, 0, 1, 1}, + {&__pyx_n_s__os, __pyx_k__os, sizeof(__pyx_k__os), 0, 0, 1, 1}, + {&__pyx_n_s__path, __pyx_k__path, sizeof(__pyx_k__path), 0, 0, 1, 1}, + {&__pyx_n_s__property, __pyx_k__property, sizeof(__pyx_k__property), 0, 0, 1, 1}, + {&__pyx_n_s__range, __pyx_k__range, sizeof(__pyx_k__range), 0, 0, 1, 1}, + {&__pyx_n_s__scores, __pyx_k__scores, sizeof(__pyx_k__scores), 0, 0, 1, 1}, + {&__pyx_n_s__sort, __pyx_k__sort, sizeof(__pyx_k__sort), 0, 0, 1, 1}, + {&__pyx_n_s__staticmethod, __pyx_k__staticmethod, sizeof(__pyx_k__staticmethod), 0, 0, 1, 1}, + {&__pyx_n_s__stem, __pyx_k__stem, sizeof(__pyx_k__stem), 0, 0, 1, 1}, + {&__pyx_n_s__top, __pyx_k__top, sizeof(__pyx_k__top), 0, 0, 1, 1}, + {&__pyx_n_s__wa, __pyx_k__wa, sizeof(__pyx_k__wa), 0, 0, 1, 1}, + {&__pyx_n_s__words, __pyx_k__words, sizeof(__pyx_k__words), 0, 0, 1, 1}, + {&__pyx_n_s__x, __pyx_k__x, sizeof(__pyx_k__x), 0, 0, 1, 1}, + {&__pyx_n_s__y, __pyx_k__y, sizeof(__pyx_k__y), 0, 0, 1, 1}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_property = __Pyx_GetName(__pyx_b, __pyx_n_s__property); if (!__pyx_builtin_property) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_n_s__staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_range = __Pyx_GetName(__pyx_b, __pyx_n_s__range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_ValueError = __Pyx_GetName(__pyx_b, __pyx_n_s__ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "binpt.pyx":14 + * return data + * elif isinstance(data, unicode): + * return data.encode('UTF-8') # <<<<<<<<<<<<<< + * raise TypeError('Cannot convert %s to string' % type(data)) + * + */ + __pyx_k_tuple_2 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_k_tuple_2); + __Pyx_INCREF(((PyObject *)__pyx_kp_s_1)); + PyTuple_SET_ITEM(__pyx_k_tuple_2, 0, ((PyObject *)__pyx_kp_s_1)); + __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_1)); + __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_2)); + + /* "binpt.pyx":52 + * + * @staticmethod + * def desc(x, y, keys = lambda r: r.scores[0]): # <<<<<<<<<<<<<< + * '''Returns the sign of keys(y) - keys(x). + * Can only be used if scores is not an empty vector as + */ + __pyx_k_tuple_16 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_k_tuple_16); + __Pyx_INCREF(((PyObject *)__pyx_n_s__x)); + PyTuple_SET_ITEM(__pyx_k_tuple_16, 0, ((PyObject *)__pyx_n_s__x)); + __Pyx_GIVEREF(((PyObject *)__pyx_n_s__x)); + __Pyx_INCREF(((PyObject *)__pyx_n_s__y)); + PyTuple_SET_ITEM(__pyx_k_tuple_16, 1, ((PyObject *)__pyx_n_s__y)); + __Pyx_GIVEREF(((PyObject *)__pyx_n_s__y)); + __Pyx_INCREF(((PyObject *)__pyx_n_s__keys)); + PyTuple_SET_ITEM(__pyx_k_tuple_16, 2, ((PyObject *)__pyx_n_s__keys)); + __Pyx_GIVEREF(((PyObject *)__pyx_n_s__keys)); + __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_16)); + __pyx_k_codeobj_17 = (PyObject*)__Pyx_PyCode_New(3, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_18, __pyx_n_s__desc, 52, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + + /* "binpt.pyx":109 + * + * @staticmethod + * def isValidBinaryTable(stem, bint wa = False): # <<<<<<<<<<<<<< + * '''This sanity check was added to the constructor, but you can access it from outside this class + * to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + */ + __pyx_k_tuple_19 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_19)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_k_tuple_19); + __Pyx_INCREF(((PyObject *)__pyx_n_s__stem)); + PyTuple_SET_ITEM(__pyx_k_tuple_19, 0, ((PyObject *)__pyx_n_s__stem)); + __Pyx_GIVEREF(((PyObject *)__pyx_n_s__stem)); + __Pyx_INCREF(((PyObject *)__pyx_n_s__wa)); + PyTuple_SET_ITEM(__pyx_k_tuple_19, 1, ((PyObject *)__pyx_n_s__wa)); + __Pyx_GIVEREF(((PyObject *)__pyx_n_s__wa)); + __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_19)); + __pyx_k_codeobj_20 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_19, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_18, __pyx_n_s__isValidBinaryTable, 109, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_20)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initbinpt(void); /*proto*/ +PyMODINIT_FUNC initbinpt(void) +#else +PyMODINIT_FUNC PyInit_binpt(void); /*proto*/ +PyMODINIT_FUNC PyInit_binpt(void) +#endif +{ + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_binpt(void)", 0); + if ( __Pyx_check_binary_version() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #ifdef __Pyx_CyFunction_USED + if (__Pyx_CyFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4(__Pyx_NAMESTR("binpt"), __pyx_methods, 0, 0, PYTHON_API_VERSION); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + #if PY_MAJOR_VERSION < 3 + Py_INCREF(__pyx_m); + #endif + __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); + if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + /*--- Initialize various global constants etc. ---*/ + if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_module_is_main_binpt) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_n_s____main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } + /*--- Builtin init code ---*/ + if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Constants init code ---*/ + if (unlikely(__Pyx_InitCachedConstants() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_5binpt_QueryResult) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + { + PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&__pyx_type_5binpt_QueryResult, "__str__"); if (unlikely(!wrapper)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) { + __pyx_wrapperbase_5binpt_11QueryResult_10__str__ = *((PyWrapperDescrObject *)wrapper)->d_base; + __pyx_wrapperbase_5binpt_11QueryResult_10__str__.doc = __pyx_doc_5binpt_11QueryResult_10__str__; + ((PyWrapperDescrObject *)wrapper)->d_base = &__pyx_wrapperbase_5binpt_11QueryResult_10__str__; + } + } + if (__Pyx_SetAttrString(__pyx_m, "QueryResult", (PyObject *)&__pyx_type_5binpt_QueryResult) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5binpt_QueryResult = &__pyx_type_5binpt_QueryResult; + if (PyType_Ready(&__pyx_type_5binpt_BinaryPhraseTable) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "BinaryPhraseTable", (PyObject *)&__pyx_type_5binpt_BinaryPhraseTable) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5binpt_BinaryPhraseTable = &__pyx_type_5binpt_BinaryPhraseTable; + /*--- Type import code ---*/ + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + + /* "binpt.pyx":3 + * from libcpp.string cimport string + * from libcpp.vector cimport vector + * import os # <<<<<<<<<<<<<< + * import cython + * + */ + __pyx_t_1 = __Pyx_Import(((PyObject *)__pyx_n_s__os), 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (PyObject_SetAttr(__pyx_m, __pyx_n_s__os, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "binpt.pyx":37 + * + * @property + * def words(self): # <<<<<<<<<<<<<< + * '''Tuple of words (as strings)''' + * return self._words + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_QueryResult, __pyx_n_s__words); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_QueryResult->tp_dict, __pyx_n_s__words, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_QueryResult); + + /* "binpt.pyx":42 + * + * @property + * def scores(self): # <<<<<<<<<<<<<< + * '''Tuple of scores (as floats)''' + * return self._scores + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_QueryResult, __pyx_n_s__scores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_QueryResult->tp_dict, __pyx_n_s__scores, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_QueryResult); + + /* "binpt.pyx":47 + * + * @property + * def wa(self): # <<<<<<<<<<<<<< + * '''Word-alignment info (as string)''' + * return self._wa + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_QueryResult, __pyx_n_s__wa); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_QueryResult->tp_dict, __pyx_n_s__wa, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_QueryResult); + + /* "binpt.pyx":52 + * + * @staticmethod + * def desc(x, y, keys = lambda r: r.scores[0]): # <<<<<<<<<<<<<< + * '''Returns the sign of keys(y) - keys(x). + * Can only be used if scores is not an empty vector as + */ + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5binpt_11QueryResult_4desc_lambda1, 0, NULL, __pyx_n_s__binpt, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_k_4 = __pyx_t_1; + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + + /* "binpt.pyx":51 + * return self._wa + * + * @staticmethod # <<<<<<<<<<<<<< + * def desc(x, y, keys = lambda r: r.scores[0]): + * '''Returns the sign of keys(y) - keys(x). + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5binpt_11QueryResult_9desc, NULL, __pyx_n_s__binpt); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_QueryResult->tp_dict, __pyx_n_s__desc, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_QueryResult); + + /* "binpt.pyx":52 + * + * @staticmethod + * def desc(x, y, keys = lambda r: r.scores[0]): # <<<<<<<<<<<<<< + * '''Returns the sign of keys(y) - keys(x). + * Can only be used if scores is not an empty vector as + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_QueryResult, __pyx_n_s__desc); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_QueryResult->tp_dict, __pyx_n_s__desc, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_QueryResult); + + /* "binpt.pyx":109 + * + * @staticmethod + * def isValidBinaryTable(stem, bint wa = False): # <<<<<<<<<<<<<< + * '''This sanity check was added to the constructor, but you can access it from outside this class + * to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5binpt_17BinaryPhraseTable_5isValidBinaryTable, NULL, __pyx_n_s__binpt); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__isValidBinaryTable, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable); + + /* "binpt.pyx":108 + * del self.__tree + * + * @staticmethod # <<<<<<<<<<<<<< + * def isValidBinaryTable(stem, bint wa = False): + * '''This sanity check was added to the constructor, but you can access it from outside this class + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__isValidBinaryTable, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable); + + /* "binpt.pyx":126 + * + * @property + * def path(self): # <<<<<<<<<<<<<< + * return self._path + * + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__path); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__path, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable); + + /* "binpt.pyx":130 + * + * @property + * def nscores(self): # <<<<<<<<<<<<<< + * return self._nscores + * + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__nscores); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__nscores, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable); + + /* "binpt.pyx":134 + * + * @property + * def wa(self): # <<<<<<<<<<<<<< + * return self._wa + * + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__wa); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__wa, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable); + + /* "binpt.pyx":138 + * + * @property + * def delimiters(self): # <<<<<<<<<<<<<< + * return self._delimiters + * + */ + __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__delimiters); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__delimiters, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable); + + /* "binpt.pyx":1 + * from libcpp.string cimport string # <<<<<<<<<<<<<< + * from libcpp.vector cimport vector + * import os + */ + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + if (PyObject_SetAttr(__pyx_m, __pyx_n_s____test__, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + __Pyx_AddTraceback("init binpt", __pyx_clineno, __pyx_lineno, __pyx_filename); + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init binpt"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* Runtime support code */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif /* CYTHON_REFNANNY */ + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { + PyObject *result; + result = PyObject_GetAttr(dict, name); + if (!result) { + if (dict != __pyx_b) { + PyErr_Clear(); + result = PyObject_GetAttr(__pyx_b, name); + } + if (!result) { + PyErr_SetObject(PyExc_NameError, name); + } + } + return result; +} + +static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyThreadState *tstate = PyThreadState_GET(); + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_Restore(type, value, tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { +#if CYTHON_COMPILING_IN_CPYTHON + PyThreadState *tstate = PyThreadState_GET(); + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#else + PyErr_Fetch(type, value, tb); +#endif +} + +#if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + if (tb == Py_None) { + Py_DECREF(tb); + tb = 0; + } + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + Py_DECREF(value); + value = type; + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + type = 0; + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif + } + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else /* Python 3+ */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (!PyExceptionClass_Check(type)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } + else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + if (!value) { + value = PyObject_CallObject(type, NULL); + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { + PyThreadState *tstate = PyThreadState_GET(); + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } + } +bad: + return; +} +#endif + +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%s() takes %s %"PY_FORMAT_SIZE_T"d positional argument%s (%"PY_FORMAT_SIZE_T"d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AS_STRING(kw_name)); + #endif +} + +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + } else { + #if PY_MAJOR_VERSION < 3 + if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { + #else + if (unlikely(!PyUnicode_Check(key))) { + #endif + goto invalid_keyword_type; + } else { + for (name = first_kw_arg; *name; name++) { + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) break; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) break; + #endif + } + if (*name) { + values[name-argnames] = value; + } else { + for (name=argnames; name != first_kw_arg; name++) { + if (**name == key) goto arg_passed_twice; + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) goto arg_passed_twice; + #endif + } + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + } + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, **name); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%s() got an unexpected keyword argument '%s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + + + +static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact) +{ + if (!type) { + PyErr_Format(PyExc_SystemError, "Missing type object"); + return 0; + } + if (none_allowed && obj == Py_None) return 1; + else if (exact) { + if (Py_TYPE(obj) == type) return 1; + } + else { + if (PyObject_TypeCheck(obj, type)) return 1; + } + PyErr_Format(PyExc_TypeError, + "Argument '%s' has incorrect type (expected %s, got %s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); + return 0; +} + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level) { + PyObject *py_import = 0; + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + py_import = __Pyx_GetAttrString(__pyx_b, "__import__"); + if (!py_import) + goto bad; + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + #if PY_VERSION_HEX >= 0x02050000 + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + /* try package relative import first */ + PyObject *py_level = PyInt_FromLong(1); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; /* try absolute import on failure */ + } + #endif + if (!module) { + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + } + } + #else + if (level>0) { + PyErr_SetString(PyExc_RuntimeError, "Relative import is not supported for Python <=2.4."); + goto bad; + } + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, NULL); + #endif +bad: + Py_XDECREF(empty_list); + Py_XDECREF(py_import); + Py_XDECREF(empty_dict); + return module; +} + +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +{ + if (op->func_doc == NULL && op->func.m_ml->ml_doc) { +#if PY_MAJOR_VERSION >= 3 + op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); +#else + op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); +#endif + } + if (op->func_doc == 0) { + Py_INCREF(Py_None); + return Py_None; + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp = op->func_doc; + if (value == NULL) + op->func_doc = Py_None; /* Mark as deleted */ + else + op->func_doc = value; + Py_INCREF(op->func_doc); + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) +{ + if (op->func_name == NULL) { +#if PY_MAJOR_VERSION >= 3 + op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); +#else + op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); +#endif + } + Py_INCREF(op->func_name); + return op->func_name; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (value == NULL || !PyUnicode_Check(value)) { +#else + if (value == NULL || !PyString_Check(value)) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = op->func_name; + Py_INCREF(value); + op->func_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) +{ + PyObject *self; + self = m->func_closure; + if (self == NULL) + self = Py_None; + Py_INCREF(self); + return self; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) +{ + if (op->func_dict == NULL) { + op->func_dict = PyDict_New(); + if (op->func_dict == NULL) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + tmp = op->func_dict; + Py_INCREF(value); + op->func_dict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(CYTHON_UNUSED __pyx_CyFunctionObject *op) +{ + PyObject* dict = PyModule_GetDict(__pyx_m); + Py_XINCREF(dict); + return dict; +} +static PyObject * +__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op) +{ + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) +{ + if (op->defaults_tuple) { + Py_INCREF(op->defaults_tuple); + return op->defaults_tuple; + } + if (op->defaults_getter) { + PyObject *res = op->defaults_getter((PyObject *) op); + if (res) { + Py_INCREF(res); + op->defaults_tuple = res; + } + return res; + } + Py_INCREF(Py_None); + return Py_None; +} +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, + {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, 0, 0, 0}, + {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#ifndef PY_WRITE_RESTRICTED /* < Py2.5 */ +#define PY_WRITE_RESTRICTED WRITE_RESTRICTED +#endif +static PyMemberDef __pyx_CyFunction_members[] = { + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CyFunctionObject, func.m_module), PY_WRITE_RESTRICTED, 0}, + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromString(m->func.m_ml->ml_name); +#else + return PyString_FromString(m->func.m_ml->ml_name); +#endif +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {__Pyx_NAMESTR("__reduce__"), (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, + PyObject *closure, PyObject *module, PyObject* code) { + __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); + if (op == NULL) + return NULL; + op->flags = flags; + op->func_weakreflist = NULL; + op->func.m_ml = ml; + op->func.m_self = (PyObject *) op; + Py_XINCREF(closure); + op->func_closure = closure; + Py_XINCREF(module); + op->func.m_module = module; + op->func_dict = NULL; + op->func_name = NULL; + op->func_doc = NULL; + op->func_classobj = NULL; + Py_XINCREF(code); + op->func_code = code; + op->defaults_pyobjects = 0; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_getter = NULL; + PyObject_GC_Track(op); + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); + Py_CLEAR(m->func.m_module); + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_code); + Py_CLEAR(m->func_classobj); + Py_CLEAR(m->defaults_tuple); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_XDECREF(pydefaults[i]); + PyMem_Free(m->defaults); + m->defaults = NULL; + } + return 0; +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + if (m->func_weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + PyObject_GC_Del(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + Py_VISIT(m->func_closure); + Py_VISIT(m->func.m_module); + Py_VISIT(m->func_dict); + Py_VISIT(m->func_name); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_code); + Py_VISIT(m->func_classobj); + Py_VISIT(m->defaults_tuple); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_VISIT(pydefaults[i]); + } + return 0; +} +static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { + Py_INCREF(func); + return func; + } + if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { + if (type == NULL) + type = (PyObject *)(Py_TYPE(obj)); + return PyMethod_New(func, + type, (PyObject *)(Py_TYPE(type))); + } + if (obj == Py_None) + obj = NULL; + return PyMethod_New(func, obj, type); +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *func_name = __Pyx_CyFunction_get_name(op); +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromFormat("", + func_name, (void *)op); +#else + return PyString_FromFormat("", + PyString_AsString(func_name), (void *)op); +#endif +} +static PyTypeObject __pyx_CyFunctionType_type = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("cython_function_or_method"), /*tp_name*/ + sizeof(__pyx_CyFunctionObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor) __Pyx_CyFunction_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ +#if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ +#else + 0, /*reserved*/ +#endif + (reprfunc) __Pyx_CyFunction_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + __Pyx_PyCFunction_Call, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags*/ + 0, /*tp_doc*/ + (traverseproc) __Pyx_CyFunction_traverse, /*tp_traverse*/ + (inquiry) __Pyx_CyFunction_clear, /*tp_clear*/ + 0, /*tp_richcompare*/ + offsetof(__pyx_CyFunctionObject, func_weakreflist), /* tp_weaklistoffse */ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_CyFunction_methods, /*tp_methods*/ + __pyx_CyFunction_members, /*tp_members*/ + __pyx_CyFunction_getsets, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + __Pyx_CyFunction_descr_get, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + offsetof(__pyx_CyFunctionObject, func_dict),/*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ +#if PY_VERSION_HEX >= 0x02060000 + 0, /*tp_version_tag*/ +#endif +}; +static int __Pyx_CyFunction_init(void) +{ + if (PyType_Ready(&__pyx_CyFunctionType_type) < 0) + return -1; + __pyx_CyFunctionType = &__pyx_CyFunctionType_type; + return 0; +} +void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyMem_Malloc(size); + if (!m->defaults) + return PyErr_NoMemory(); + memset(m->defaults, 0, sizeof(size)); + m->defaults_pyobjects = pyobjects; + return m->defaults; +} +static void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} + +static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { + const unsigned char neg_one = (unsigned char)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(unsigned char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned char)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to unsigned char" : + "value too large to convert to unsigned char"); + } + return (unsigned char)-1; + } + return (unsigned char)val; + } + return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); +} + +static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { + const unsigned short neg_one = (unsigned short)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(unsigned short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned short)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to unsigned short" : + "value too large to convert to unsigned short"); + } + return (unsigned short)-1; + } + return (unsigned short)val; + } + return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); +} + +static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { + const unsigned int neg_one = (unsigned int)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(unsigned int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned int)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to unsigned int" : + "value too large to convert to unsigned int"); + } + return (unsigned int)-1; + } + return (unsigned int)val; + } + return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); +} + +static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject* x) { + const char neg_one = (char)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(char)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to char" : + "value too large to convert to char"); + } + return (char)-1; + } + return (char)val; + } + return (char)__Pyx_PyInt_AsLong(x); +} + +static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject* x) { + const short neg_one = (short)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(short)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to short" : + "value too large to convert to short"); + } + return (short)-1; + } + return (short)val; + } + return (short)__Pyx_PyInt_AsLong(x); +} + +static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject* x) { + const int neg_one = (int)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(int)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to int" : + "value too large to convert to int"); + } + return (int)-1; + } + return (int)val; + } + return (int)__Pyx_PyInt_AsLong(x); +} + +static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { + const signed char neg_one = (signed char)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(signed char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed char)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to signed char" : + "value too large to convert to signed char"); + } + return (signed char)-1; + } + return (signed char)val; + } + return (signed char)__Pyx_PyInt_AsSignedLong(x); +} + +static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { + const signed short neg_one = (signed short)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(signed short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed short)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to signed short" : + "value too large to convert to signed short"); + } + return (signed short)-1; + } + return (signed short)val; + } + return (signed short)__Pyx_PyInt_AsSignedLong(x); +} + +static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { + const signed int neg_one = (signed int)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(signed int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed int)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to signed int" : + "value too large to convert to signed int"); + } + return (signed int)-1; + } + return (signed int)val; + } + return (signed int)__Pyx_PyInt_AsSignedLong(x); +} + +static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject* x) { + const int neg_one = (int)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; + if (sizeof(int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(int)val)) { + if (!unlikely(val == -1 && PyErr_Occurred())) { + PyErr_SetString(PyExc_OverflowError, + (is_unsigned && unlikely(val < 0)) ? + "can't convert negative value to int" : + "value too large to convert to int"); + } + return (int)-1; + } + return (int)val; + } + return (int)__Pyx_PyInt_AsLong(x); +} + +static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { + const unsigned long neg_one = (unsigned long)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return (unsigned long)val; + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return (unsigned long)PyLong_AsUnsignedLong(x); + } else { + return (unsigned long)PyLong_AsLong(x); + } + } else { + unsigned long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned long)-1; + val = __Pyx_PyInt_AsUnsignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { + const unsigned PY_LONG_LONG neg_one = (unsigned PY_LONG_LONG)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return (unsigned PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return (unsigned PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); + } else { + return (unsigned PY_LONG_LONG)PyLong_AsLongLong(x); + } + } else { + unsigned PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsUnsignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { + const long neg_one = (long)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long)-1; + } + return (long)val; + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long)-1; + } + return (long)PyLong_AsUnsignedLong(x); + } else { + return (long)PyLong_AsLong(x); + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (long)-1; + val = __Pyx_PyInt_AsLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { + const PY_LONG_LONG neg_one = (PY_LONG_LONG)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to PY_LONG_LONG"); + return (PY_LONG_LONG)-1; + } + return (PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to PY_LONG_LONG"); + return (PY_LONG_LONG)-1; + } + return (PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); + } else { + return (PY_LONG_LONG)PyLong_AsLongLong(x); + } + } else { + PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { + const signed long neg_one = (signed long)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to signed long"); + return (signed long)-1; + } + return (signed long)val; + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to signed long"); + return (signed long)-1; + } + return (signed long)PyLong_AsUnsignedLong(x); + } else { + return (signed long)PyLong_AsLong(x); + } + } else { + signed long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed long)-1; + val = __Pyx_PyInt_AsSignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { + const signed PY_LONG_LONG neg_one = (signed PY_LONG_LONG)-1, const_zero = 0; + const int is_unsigned = neg_one > const_zero; +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to signed PY_LONG_LONG"); + return (signed PY_LONG_LONG)-1; + } + return (signed PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to signed PY_LONG_LONG"); + return (signed PY_LONG_LONG)-1; + } + return (signed PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); + } else { + return (signed PY_LONG_LONG)PyLong_AsLongLong(x); + } + } else { + signed PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsSignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + #if PY_VERSION_HEX < 0x02050000 + return PyErr_Warn(NULL, message); + #else + return PyErr_WarnEx(NULL, message, 1); + #endif + } + return 0; +} + +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = (start + end) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, /*int argcount,*/ + 0, /*int kwonlyargcount,*/ + 0, /*int nlocals,*/ + 0, /*int stacksize,*/ + 0, /*int flags,*/ + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, /*int firstlineno,*/ + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_globals = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_globals = PyModule_GetDict(__pyx_m); + if (!py_globals) goto bad; + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + py_globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = py_line; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else /* Python 3+ has unicode identifiers */ + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + + +/* Type Conversion Functions */ + +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} + +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { + PyNumberMethods *m; + const char *name = NULL; + PyObject *res = NULL; +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return Py_INCREF(x), x; + m = Py_TYPE(x)->tp_as_number; +#if PY_VERSION_HEX < 0x03000000 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } +#else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } +#endif + if (res) { +#if PY_VERSION_HEX < 0x03000000 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%s__ returned non-%s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} + +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject* x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} + +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { +#if PY_VERSION_HEX < 0x02050000 + if (ival <= LONG_MAX) + return PyInt_FromLong((long)ival); + else { + unsigned char *bytes = (unsigned char *) &ival; + int one = 1; int little = (int)*(unsigned char*)&one; + return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); + } +#else + return PyInt_FromSize_t(ival); +#endif +} + +static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { + unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); + if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { + return (size_t)-1; + } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t)-1; + } + return (size_t)val; +} + + +#endif /* Py_PYTHON_H */ diff --git a/contrib/python/binpt/binpt.pxd b/contrib/python/binpt/binpt.pxd new file mode 100644 index 000000000..33661ceaf --- /dev/null +++ b/contrib/python/binpt/binpt.pxd @@ -0,0 +1,25 @@ +from libcpp.string cimport string +from libcpp.vector cimport vector +from libcpp.pair cimport pair + +ctypedef string* str_pointer + +cdef extern from 'TypeDef.h' namespace 'Moses': + ctypedef vector[float] Scores + ctypedef pair[vector[str_pointer], Scores] StringTgtCand + +cdef extern from 'PhraseDictionaryTree.h' namespace 'Moses': + cdef cppclass PhraseDictionaryTree: + PhraseDictionaryTree(unsigned nscores) + void UseWordAlignment(bint use) + bint UseWordAlignment() + int Read(string& path) + void GetTargetCandidates(vector[string]& fs, + vector[StringTgtCand]& rv) + void GetTargetCandidates(vector[string]& fs, + vector[StringTgtCand]& rv, + vector[string]& wa) + +cdef extern from 'Util.h' namespace 'Moses': + cdef vector[string] Tokenize(string& text, string& delimiters) + diff --git a/contrib/python/binpt/binpt.pyx b/contrib/python/binpt/binpt.pyx new file mode 100644 index 000000000..e66981df6 --- /dev/null +++ b/contrib/python/binpt/binpt.pyx @@ -0,0 +1,166 @@ +from libcpp.string cimport string +from libcpp.vector cimport vector +import os +import cython + +cpdef int fsign(float x): + '''Simply returns the sign of float x (zero is assumed +), it's defined here just so one gains a little bit with static typing''' + return 1 if x >= 0 else -1 + +cdef bytes as_str(data): + if isinstance(data, bytes): + return data + elif isinstance(data, unicode): + return data.encode('UTF-8') + raise TypeError('Cannot convert %s to string' % type(data)) + +cdef class QueryResult(object): + '''This class represents a query result, that is, + a target phrase (tuple of words/strings), + a feature vector (tuple of floats) + and possibly an alignment info (string). + Here we don't bother parsing the alignment info, as it's often only + used as is, threfore saving some time.''' + + cdef tuple _words + cdef tuple _scores + cdef bytes _wa + + def __cinit__(self, words, scores, wa = None): + '''Requires a tuple of words (as strings) and a tuple of scores (as floats). + Word-alignment info (as string) may be provided''' + self._words = words + self._scores = scores + self._wa = wa + + @property + def words(self): + '''Tuple of words (as strings)''' + return self._words + + @property + def scores(self): + '''Tuple of scores (as floats)''' + return self._scores + + @property + def wa(self): + '''Word-alignment info (as string)''' + return self._wa + + @staticmethod + def desc(x, y, keys = lambda r: r.scores[0]): + '''Returns the sign of keys(y) - keys(x). + Can only be used if scores is not an empty vector as + keys defaults to scores[0]''' + return fsign(keys(y) - keys(x)) + + def __str__(self): + '''Returns a string such as: ||| [||| word-alignment info]''' + if self._wa: + return ' ||| '.join( (' '.join(self._words), + ' '.join([str(x) for x in self._scores]), + self._wa) ) + else: + return ' ||| '.join( (' '.join(self._words), + ' '.join([str(x) for x in self._scores]) ) ) + + def __repr__(self): + return repr((repr(self._words), repr(self._scores), repr(self._wa))) + +cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None): + '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string) + to a QueryResult (python object).''' + cdef tuple words = tuple([cand.first[i].c_str() for i in range(cand.first.size())]) + cdef tuple scores = tuple([cand.second[i] for i in range(cand.second.size())]) + return QueryResult(words, scores, wa) + +cdef class BinaryPhraseTable(object): + '''This class encapsulates a Moses::PhraseDictionaryTree for operations over + binary phrase tables.''' + + cdef PhraseDictionaryTree* __tree + cdef bytes _path + cdef unsigned _nscores + cdef bint _wa + cdef bytes _delimiters + + def __cinit__(self, bytes path, unsigned nscores = 5, bint wa = False, delimiters = ' \t'): + '''It requies a path to binary phrase table (stem of the table, e.g europarl.fr-en + is the stem for europar.fr-en.binphr.*). + Moses::PhraseDictionaryTree also needs to be aware of the number of scores (usually 5), + and whether or not there is word-alignment info in the table (usually not). + One can also specify the token delimiters, for Moses::Tokenize(text, delimiters), which is space or tab by default.''' + + if not BinaryPhraseTable.isValidBinaryTable(path, wa): + raise ValueError, "'%s' doesn't seem a valid binary table." % path + self._path = path + self._nscores = nscores + self._wa = wa + self._delimiters = delimiters + self.__tree = new PhraseDictionaryTree(nscores) + self.__tree.UseWordAlignment(wa) + self.__tree.Read(string(path)) + + def __dealloc__(self): + del self.__tree + + @staticmethod + def isValidBinaryTable(stem, bint wa = False): + '''This sanity check was added to the constructor, but you can access it from outside this class + to determine whether or not you are providing a valid stem to BinaryPhraseTable.''' + if wa: + return os.path.isfile(stem + ".binphr.idx") \ + and os.path.isfile(stem + ".binphr.srctree.wa") \ + and os.path.isfile(stem + ".binphr.srcvoc") \ + and os.path.isfile(stem + ".binphr.tgtdata.wa") \ + and os.path.isfile(stem + ".binphr.tgtvoc") + else: + return os.path.isfile(stem + ".binphr.idx") \ + and os.path.isfile(stem + ".binphr.srctree") \ + and os.path.isfile(stem + ".binphr.srcvoc") \ + and os.path.isfile(stem + ".binphr.tgtdata") \ + and os.path.isfile(stem + ".binphr.tgtvoc") + + @property + def path(self): + return self._path + + @property + def nscores(self): + return self._nscores + + @property + def wa(self): + return self._wa + + @property + def delimiters(self): + return self._delimiters + + def query(self, line, cmp = None, top = 0): + '''Queries the phrase table and returns a list of matches. + Each match is a QueryResult. + If 'cmp' is defined the return list is sorted. + If 'top' is defined, onlye the top elements will be returned.''' + cdef bytes text = as_str(line) + cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters)) + cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]() + cdef vector[string]* wa = NULL + cdef list phrases + if not self.__tree.UseWordAlignment(): + self.__tree.GetTargetCandidates(fphrase, rv[0]) + phrases = [get_query_result(rv[0][i]) for i in range(rv.size())] + else: + wa = new vector[string]() + self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0]) + phrases = [get_query_result(rv[0][i], wa[0][i].c_str()) for i in range(rv.size())] + del wa + del rv + if cmp: + phrases.sort(cmp=cmp) + if top > 0: + return phrases[0:top] + else: + return phrases + diff --git a/contrib/python/example.py b/contrib/python/example.py new file mode 100644 index 000000000..8494ba5fe --- /dev/null +++ b/contrib/python/example.py @@ -0,0 +1,31 @@ +import binpt +#from binpt import QueryResult +import sys + + +if len(sys.argv) < 3: + print "Usage: %s phrase-table nscores [wa] < query > result" % (sys.argv[0]) + sys.exit(0) + +pt_file = sys.argv[1] +nscores = int(sys.argv[2]) +wa = len(sys.argv) == 4 + +pt = binpt.BinaryPhraseTable(pt_file, nscores, wa) +print >> sys.stderr, "-ttable %s -nscores %d -alignment-info %s -delimiter '%s'\n" %(pt.path, pt.nscores, str(pt.wa), pt.delimiters) + +for line in sys.stdin: + f = line.strip() + matches = pt.query(f, cmp = binpt.QueryResult.desc, top = 20) + print '\n'.join([' ||| '.join((f, str(e))) for e in matches]) + ''' + # This is how one would use the QueryResult object + for e in matches: + print ' '.join(e.words) # tuple of strings + print e.scores # tuple of floats + if e.wa: + print e.wa # string + ''' + + + diff --git a/contrib/python/examples/phrase-table.binphr.idx b/contrib/python/examples/phrase-table.binphr.idx new file mode 100644 index 000000000..58adc514e Binary files /dev/null and b/contrib/python/examples/phrase-table.binphr.idx differ diff --git a/contrib/python/examples/phrase-table.binphr.srctree.wa b/contrib/python/examples/phrase-table.binphr.srctree.wa new file mode 100644 index 000000000..a6da5e1bf Binary files /dev/null and b/contrib/python/examples/phrase-table.binphr.srctree.wa differ diff --git a/contrib/python/examples/phrase-table.binphr.srcvoc b/contrib/python/examples/phrase-table.binphr.srcvoc new file mode 100644 index 000000000..d8656e003 --- /dev/null +++ b/contrib/python/examples/phrase-table.binphr.srcvoc @@ -0,0 +1,2 @@ +1 essa +0 casa diff --git a/contrib/python/examples/phrase-table.binphr.tgtdata.wa b/contrib/python/examples/phrase-table.binphr.tgtdata.wa new file mode 100644 index 000000000..592874362 Binary files /dev/null and b/contrib/python/examples/phrase-table.binphr.tgtdata.wa differ diff --git a/contrib/python/examples/phrase-table.binphr.tgtvoc b/contrib/python/examples/phrase-table.binphr.tgtvoc new file mode 100644 index 000000000..71975c3c5 --- /dev/null +++ b/contrib/python/examples/phrase-table.binphr.tgtvoc @@ -0,0 +1,4 @@ +3 this +2 location +1 house +0 building diff --git a/contrib/python/examples/phrase-table.txt b/contrib/python/examples/phrase-table.txt new file mode 100644 index 000000000..1b2a2630a --- /dev/null +++ b/contrib/python/examples/phrase-table.txt @@ -0,0 +1,4 @@ +casa ||| building ||| 0.6 0.75 0.35 0.35 2.718 ||| 0-0 ||| 2 2 +casa ||| house ||| 0.7 0.75 0.35 0.35 2.718 ||| 0-0 ||| 2 2 +casa ||| location ||| 0.5 0.75 0.35 0.35 2.718 ||| 0-0 ||| 2 2 +essa casa ||| this house ||| 0.7 0.5 0.8 0.6 2.718 ||| 0-0 1-1 ||| 2 2 diff --git a/contrib/python/setup.py b/contrib/python/setup.py new file mode 100644 index 000000000..66042fbc8 --- /dev/null +++ b/contrib/python/setup.py @@ -0,0 +1,47 @@ +from distutils.core import setup +from distutils.extension import Extension +import os +import sys + +available_switches = ['--with-cmph'] +with_cmph = False + +while sys.argv[-1] in available_switches: + switch = sys.argv.pop() + if switch == '--with-cmph': + with_cmph = True + + +#### From here you probably don't need to change anything +#### unless a new dependency shows up in Moses + +mosesdir = os.path.abspath('../../') +includes = [mosesdir, os.path.join(mosesdir, 'moses/src'), os.path.join(mosesdir, 'util')] +libdir = os.path.join(mosesdir, 'lib') + +basic=['z', 'stdc++', 'pthread', 'm', 'gcc_s', 'c', 'boost_system', 'boost_thread', 'boost_filesystem', 'rt'] +moses=['OnDiskPt', 'kenutil', 'kenlm', 'LM', 'mert_lib', 'moses_internal', 'CYKPlusParser', 'Scope3Parser', 'fuzzy-match', 'RuleTable', 'CompactPT', 'moses', 'dynsa', 'pcfg_common' ] +additional=[] + +if with_cmph: + additional.append('cmph') + +exobj = [os.path.join(libdir, 'lib' + l + '.so') for l in moses] + +ext_modules = [ + Extension(name = 'binpt', + sources = ['binpt/binpt.cpp'], + language = 'C++', + include_dirs = includes, + extra_objects = exobj, + library_dirs = [libdir], + runtime_library_dirs = [libdir], + libraries = basic + moses + additional, + extra_compile_args = ['-O3', '-DNDEBUG'], + ) +] + +setup( + name='binpt', + ext_modules=ext_modules +) diff --git a/contrib/relent-filter/AUTHORS b/contrib/relent-filter/AUTHORS new file mode 100644 index 000000000..184a6dddd --- /dev/null +++ b/contrib/relent-filter/AUTHORS @@ -0,0 +1 @@ +Wang Ling - lingwang at cs dot cmu dot edu diff --git a/contrib/relent-filter/README.txt b/contrib/relent-filter/README.txt new file mode 100644 index 000000000..e791d1f8a --- /dev/null +++ b/contrib/relent-filter/README.txt @@ -0,0 +1,91 @@ +Implementation of the Relative Entropy-based Phrase table filtering algorithm by Wang Ling (Ling et al, 2012). + +This implementation also calculates the significance scores for the phrase tables based on the Fisher's Test(Johnson et al, 2007). Uses a slightly modified version of the "sigtest-filter" by Chris Dyer. + +-------BUILD INSTRUCTIONS------- + +1 - Build the sigtest-filter binary + +1.1 - Download and build SALM available at http://projectile.sv.cmu.edu/research/public/tools/salm/salm.htm + +1.2 - Run "make SALMDIR=" in "/contrib/relent-filter/sigtest-filter" to create the executable filter-pt + +2 - Build moses project by running "./bjam ", this will create the executables for relent filtering + +-------USAGE INSTRUCTIONS------- + +Required files: +s_train - source training file +t_train - target training file +moses_ini - path to the moses configuration file ( after tuning ) +pruning_binaries - path to the relent pruning binaries ( should be "/bin" ) +pruning_scripts - path to the relent pruning scripts ( should be "/contrib/relent-filter/scripts" ) +sigbin - path to the sigtest filter binaries ( should be "/contrib/relent-filter/sigtest-filter" ) +output_dir - path to write the output + +1 - build suffix arrays for the source and target parallel training data + +1.1 - run "/Bin/Linux/Index/IndexSA.O32 " (or IndexSA.O64) + +1.2 - run "/Bin/Linux/Index/IndexSA.O32 " (or IndexSA.O64) + +2 - calculate phrase pair scores by running: + +perl /calcPruningScores.pl -moses_ini -training_s -training_t -prune_bin -prune_scripts -moses_scripts /scripts/training/ -workdir -dec_size 10000 + +this will create the following files in the dir: + +count.txt - counts of the phrase pairs for N(s,t) N(s,*) and N(*,t) +divergence.txt - negative log of the divergence of the phrase pair +empirical.txt - empirical distribution of the phrase pairs N(s,t)/N(*,*) +rel_ent.txt - relative entropy of the phrase pairs +significance.txt - significance of the phrase pairs + +You can use any one of these files for pruning and also combine these scores using /interpolateScores.pl + +3 - To actually prune a phrase table you should run /prunePT.pl + +For instance, to prune 30% of the phrase table using rel_ent run: +perl /prunePT.pl -table -scores /scores/rel_ent.txt -percentage 70 > + +You can also prune by threshold +perl /prunePT.pl -table -scores /scores/rel_ent.txt -threshold 0.1 > + +The same must be done for the reordering table by replacing with the + +perl /prunePT.pl -table -scores /scores/rel_ent.txt -percentage 70 > + +-------RUNNING STEP 2 IN PARALLEL------- + +Step 2 requires the forced decoding of the whole set of phrase pairs in the table, so unless you test it on a small corpora, it usually requires large amounts of time to process. +Thus, we recommend users to run multiple instances of "/calcPruningScores.pl" in parallel to process different parts of the phrase table. + +To do this, run: + +perl /calcPruningScores.pl -moses_ini -training_s -training_t -prune_bin -prune_scripts -moses_scripts /scripts/training/ -workdir -dec_size 10000 -start 0 -end 100000 + +The -start and -end tags tell the script to only calculate the results for phrase pairs between 0 and 99999. + +Thus, an example of a shell script to run for the whole phrase table would be: + +size=`wc | gawk '{print $1}'` +phrases_per_process=100000 + +for i in $(seq 0 $phrases_per_process $size) +do + end=`expr $i + $phrases_per_process` + perl /calcPruningScores.pl -moses_ini -training_s -training_t -prune_bin -prune_scripts -moses_scripts /scripts/training/ -workdir .$i-$end -dec_size 10000 -start $i -end $end +done + +After all processes finish, simply join the partial score files together in the same order. + +-------REFERENCES------- +Ling, W., Graça, J., Trancoso, I., and Black, A. (2012). Entropy-based pruning for phrase-based +machine translation. In Proceedings of the 2012 +Joint Conference on Empirical Methods in Natural Language Processing and +Computational Natural Language Learning (EMNLP-CoNLL), pp. 962-971. + +H. Johnson, J. Martin, G. Foster and R. Kuhn. (2007) Improving Translation +Quality by Discarding Most of the Phrasetable. In Proceedings of the 2007 +Joint Conference on Empirical Methods in Natural Language Processing and +Computational Natural Language Learning (EMNLP-CoNLL), pp. 967-975. diff --git a/contrib/relent-filter/scripts/calcEmpiricalDistribution.pl b/contrib/relent-filter/scripts/calcEmpiricalDistribution.pl new file mode 100644 index 000000000..462ec5339 --- /dev/null +++ b/contrib/relent-filter/scripts/calcEmpiricalDistribution.pl @@ -0,0 +1,53 @@ +#!/usr/bin/perl -w + +# read arguments +my $countFile = $ARGV[0]; + +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; + +&process_count_file($countFile); + +sub process_count_file { + $file = $_[0]; + open(COUNT_READER, &open_compressed($file)) or die "ERROR: Can't read $file"; + + print STDERR "reading file to calculate normalizer"; + $normalizer=0; + while() { + my $line = $_; + chomp($line); + my @line_array = split(/\s+/, $line); + my $count = $line_array[0]; + $normalizer+=$count; + } + + close(COUNT_READER); + + print STDERR "reading file again to print the counts"; + open(COUNT_READER, &open_compressed($file)) or die "ERROR: Can't read $file"; + + while() { + my $line = $_; + chomp($line); + my @line_array = split(/\s+/, $line); + my $score = $line_array[0]/$normalizer; + print $score."\n"; + } + + close(COUNT_READER); +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} diff --git a/contrib/relent-filter/scripts/calcPruningScores.pl b/contrib/relent-filter/scripts/calcPruningScores.pl new file mode 100755 index 000000000..cbfabac55 --- /dev/null +++ b/contrib/relent-filter/scripts/calcPruningScores.pl @@ -0,0 +1,351 @@ +#!/usr/bin/perl -w +use Getopt::Long; +use File::Basename; +use POSIX; + +# read arguments +my $line_start = 0; +my $line_end = LONG_MAX; +my $tmp_dir = ""; +my $dec_size = LONG_MAX; +$_HELP = 1 if (@ARGV < 1 or !GetOptions ("moses_ini=s" => \$moses_ini, #moses conf file +"start:i" => \$line_start, #fisrt phrase to process +"end:i" => \$line_end, #last sentence to process (not including) +"training_s=s" => \$training_s, #source training file +"training_t=s" => \$training_t, #target training file +"prune_bin=s" => \$prune_bin, #binary files in the pruning toolkit +"prune_scripts=s" => \$prune_scripts, #scripts in the pruning toolkit +"sig_bin=s" => \$sig_bin, #binary files to calculate significance +"moses_scripts=s" => \$moses_scripts, #dir with the moses scripts +"tmp_dir:s" => \$tmp_dir, #dir with the moses scripts +"dec_size:i" => \$dec_size, #dir with the moses scripts +"workdir=s" => \$workdir)); #directory to put all the output files + +# help message if arguments are not correct +if ($_HELP) { + print " +Usage: perl calcPruningScores.pl [PARAMS] +Function: Calculates relative entropy for each phrase pair in a translation model. +Authors: Wang Ling ( lingwang at cs dot cmu dot edu ) +PARAMS: + -moses_ini : moses configuration file with the model to prune (phrase table, reordering table, weights etc...) + -training_s : source training file, please run salm first + -training_t : target training file, please run salm first + -prune_bin : path to the binaries for pruning (probably /bin) + -prune_scripts : path to the scripts for pruning (probably the directory where this script is) + -sig_bin : path to the binary for significance testing included in this toolkit + -moses_scripts : path to the moses training scripts (where filter-model-given-input.pl is) + -workdir : directory to produce the output + -tmp_dir : directory to store temporary files (improve performance if stored in a local disk), omit to store in workdir + -dec_size : number of phrase pairs to be decoded at a time, omit to decode all selected phrase pairs at once + -start and -end : starting and ending phrase pairs to process, to be used if you want to launch multiple processes in parallel for different parts of the phrase table. If specified the process will process the phrase pairs from to + +For any questions contact lingwang at cs dot cmu dot edu +"; + exit(1); +} + +# setting up working dirs +my $TMP_DIR = $tmp_dir; +if ($tmp_dir eq ""){ + $TMP_DIR = "$workdir/tmp"; +} +my $SCORE_DIR = "$workdir/scores"; +my $FILTER_DIR = "$TMP_DIR/filter"; + +# files for divergence module +my $SOURCE_FILE = "$TMP_DIR/source.txt"; +my $CONSTRAINT_FILE = "$TMP_DIR/constraint.txt"; +my $DIVERGENCE_FILE = "$SCORE_DIR/divergence.txt"; + +# files for significance module +my $SIG_TABLE_FILE = "$TMP_DIR/source_target.txt"; +my $SIG_MOD_OUTPUT = "$TMP_DIR/sig_mod.out"; +my $SIG_FILE = "$SCORE_DIR/significance.txt"; +my $COUNT_FILE = "$SCORE_DIR/count.txt"; +my $EMP_DIST_FILE= "$SCORE_DIR/empirical.txt"; +my $REL_ENT_FILE= "$SCORE_DIR/rel_ent.txt"; + +# setting up executables +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; +my $CP = "cp"; +my $SED = "sed"; +my $RM = "rm"; +my $SORT_EXEC = "sort"; +my $PRUNE_EXEC = "$prune_bin/calcDivergence"; +my $SIG_EXEC = "$sig_bin/filter-pt"; +my $FILTER_EXEC = "perl $moses_scripts/filter-model-given-input.pl"; +my $CALC_EMP_EXEC ="perl $prune_scripts/calcEmpiricalDistribution.pl"; +my $INT_TABLE_EXEC = "perl $prune_scripts/interpolateScores.pl"; + +# moses ini variables +my ($TRANSLATION_TABLE_FILE, $REORDERING_TABLE_FILE); + +# phrase table variables +my ($N_PHRASES, $N_PHRASES_TO_PROCESS); + +# main functions +&prepare(); +&calc_sig_and_counts(); +&calc_div(); +&clear_up(); + +# (1) preparing data +sub prepare { + print STDERR "(1) preparing data @ ".`date`; + safesystem("mkdir -p $workdir") or die("ERROR: could not create work dir $workdir"); + safesystem("mkdir -p $TMP_DIR") or die("ERROR: could not create work dir $TMP_DIR"); + safesystem("mkdir -p $SCORE_DIR") or die("ERROR: could not create work dir $SCORE_DIR"); + &get_moses_ini_params(); + ©_tables_to_tmp_dir(); + &write_data_files(); + + $N_PHRASES = &get_number_of_phrases(); + $line_end = ($line_end > $N_PHRASES) ? $N_PHRASES : $line_end; + $N_PHRASES_TO_PROCESS = $line_end - $line_start; +} + +sub write_data_files { + open(SOURCE_WRITER,">".$SOURCE_FILE) or die "ERROR: Can't write $SOURCE_FILE"; + open(CONSTRAINT_WRITER,">".$CONSTRAINT_FILE) or die "ERROR: Can't write $CONSTRAINT_FILE"; + open(TABLE_WRITER,">".$SIG_TABLE_FILE) or die "ERROR: Can't write $SIG_TABLE_FILE"; + open(TTABLE_READER, &open_compressed($TRANSLATION_TABLE_FILE)) or die "ERROR: Can't read $TRANSLATION_TABLE_FILE"; + + $line_number = 0; + while($line_number < $line_start && !eof(TTABLE_READER)){ + ; + $line_number++; + } + while($line_number < $line_end && !eof(TTABLE_READER)) { + my $line = ; + chomp($line); + my @line_array = split(/\s+\|\|\|\s+/, $line); + my $source = $line_array[0]; + my $target = $line_array[1]; + my $scores = $line_array[2]; + print TABLE_WRITER $source." ||| ".$target." ||| ".$scores."\n"; + print SOURCE_WRITER $source."\n"; + print CONSTRAINT_WRITER $target."\n"; + $line_number++; + } + + close(SOURCE_WRITER); + close(CONSTRAINT_WRITER); + close(TABLE_WRITER); + close(TTABLE_READER); +} + +sub copy_tables_to_tmp_dir { + $tmp_t_table = "$TMP_DIR/".basename($TRANSLATION_TABLE_FILE); + $tmp_r_table = "$TMP_DIR/".basename($REORDERING_TABLE_FILE); + $tmp_moses_ini = "$TMP_DIR/moses.ini"; + $cp_t_cmd = "$CP $TRANSLATION_TABLE_FILE $TMP_DIR"; + $cp_r_cmd = "$CP $REORDERING_TABLE_FILE $TMP_DIR"; + safesystem("$cp_t_cmd") or die("ERROR: could not run:\n $cp_t_cmd"); + safesystem("$cp_r_cmd") or die("ERROR: could not run:\n $cp_r_cmd"); + + $sed_cmd = "$SED s#$TRANSLATION_TABLE_FILE#$tmp_t_table#g $moses_ini | $SED s#$REORDERING_TABLE_FILE#$tmp_r_table#g > $tmp_moses_ini"; + safesystem("$sed_cmd") or die("ERROR: could not run:\n $sed_cmd"); + + $TRANSLATION_TABLE_FILE = $tmp_t_table; + $REORDERING_TABLE_FILE = $tmp_r_table; + $moses_ini = $tmp_moses_ini; +} + +# (2) calculating sig and counts +sub calc_sig_and_counts { + print STDERR "(2) calculating counts and significance".`date`; + print STDERR "(2.1) running significance module".`date`; + &run_significance_module(); + print STDERR "(2.2) writing counts and significance tables".`date`; + &write_counts_and_significance_table(); + print STDERR "(2.3) calculating empirical distribution".`date`; +} + +sub write_counts_and_significance_table { + open(COUNT_WRITER,">".$COUNT_FILE) or die "ERROR: Can't write $COUNT_FILE"; + open(SIG_WRITER,">".$SIG_FILE) or die "ERROR: Can't write $SIG_FILE"; + open(SIG_MOD_READER, &open_compressed($SIG_MOD_OUTPUT)) or die "ERROR: Can't read $SIG_MOD_OUTPUT"; + + while() { + my($line) = $_; + chomp($line); + my @line_array = split(/\s+\|\|\|\s+/, $line); + my $count = $line_array[0]; + my $sig = $line_array[1]; + print COUNT_WRITER $count."\n"; + print SIG_WRITER $sig."\n"; + } + + close(SIG_MOD_READER); + close(COUNT_WRITER); + close(SIG_WRITER); +} + +sub run_significance_module { + my $sig_cmd = "cat $SIG_TABLE_FILE | $SIG_EXEC -e $training_t -f $training_s -l -10000 -p -c > $SIG_MOD_OUTPUT"; + safesystem("$sig_cmd") or die("ERROR: could not run:\n $sig_cmd"); +} + +# (3) calculating divergence +sub calc_div { + print STDERR "(3) calculating relative entropy".`date`; + print STDERR "(3.1) calculating empirical distribution".`date`; + &calculate_empirical_distribution(); + print STDERR "(3.2) calculating divergence (this might take a while)".`date`; + if($N_PHRASES_TO_PROCESS > $dec_size) { + &calculate_divergence_shared("$FILTER_DIR"); + } + else{ + &calculate_divergence($moses_ini); + } + print STDERR "(3.3) calculating relative entropy from empirical and divergence distributions".`date`; + &calculate_relative_entropy(); +} + +sub calculate_empirical_distribution { + my $emp_cmd = "$CALC_EMP_EXEC $COUNT_FILE > $EMP_DIST_FILE"; + safesystem("$emp_cmd") or die("ERROR: could not run:\n $emp_cmd"); +} + +sub get_fragmented_file_name { + my ($name, $frag, $interval) = @_; + return "$name-$frag-".($frag+$interval); +} + +sub calculate_divergence { + my $moses_ini_file = $_[0]; + print STDERR "force decoding phrase pairs\n"; + my $prune_cmd = "cat $SOURCE_FILE | $PRUNE_EXEC -f $moses_ini_file -constraint $CONSTRAINT_FILE -early-discarding-threshold 0 -s 100000 -ttable-limit 0 > $DIVERGENCE_FILE 2> /dev/null"; + safesystem("$prune_cmd") or die("ERROR: could not run:\n $prune_cmd"); +} + +sub calculate_divergence_shared { + my $filter_dir = $_[0]; + + &split_file_into_chunks($SOURCE_FILE, $dec_size, $N_PHRASES_TO_PROCESS); + &split_file_into_chunks($CONSTRAINT_FILE, $dec_size, $N_PHRASES_TO_PROCESS); + + for(my $i = 0; $i < $N_PHRASES_TO_PROCESS; $i = $i + $dec_size) { + my $filter_cmd = "$FILTER_EXEC ".&get_fragmented_file_name($FILTER_DIR, $i, $dec_size)." $moses_ini ".&get_fragmented_file_name($SOURCE_FILE, $i, $dec_size); + safesystem("$filter_cmd") or die("ERROR: could not run:\n $filter_cmd"); + + my $moses_ini_file = &get_fragmented_file_name($filter_dir, $i, $dec_size)."/moses.ini"; + my $source_file = &get_fragmented_file_name($SOURCE_FILE, $i, $dec_size); + my $constraint_file = &get_fragmented_file_name($CONSTRAINT_FILE, $i, $dec_size); + my $prune_cmd; + print STDERR "force decoding phrase pairs $i to ".($i + $dec_size)."\n"; + if($i == 0){ + $prune_cmd = "cat $source_file | $PRUNE_EXEC -f $moses_ini_file -constraint $constraint_file -early-discarding-threshold 0 -s 100000 -ttable-limit 0 > $DIVERGENCE_FILE 2> /dev/null"; + } + else{ + $prune_cmd = "cat $source_file | $PRUNE_EXEC -f $moses_ini_file -constraint $constraint_file -early-discarding-threshold 0 -s 100000 -ttable-limit 0 >> $DIVERGENCE_FILE 2> /dev/null"; + } + safesystem("$prune_cmd") or die("ERROR: could not run:\n $prune_cmd"); + + my $rm_cmd = "$RM -r ".&get_fragmented_file_name($FILTER_DIR, $i, $dec_size); + safesystem("$rm_cmd") or die("ERROR: could not run:\n $rm_cmd"); + + } +} + +sub calculate_relative_entropy { + my $int_cmd = "$INT_TABLE_EXEC -files \"$EMP_DIST_FILE $DIVERGENCE_FILE\" -weights \"1 1\" -operation \"*\" > $REL_ENT_FILE"; + safesystem("$int_cmd") or die("ERROR: could not run:\n $int_cmd"); + +} + +# (4) clear up stuff that is not needed +sub clear_up { + print STDERR "(4) removing tmp dir".`date`; + $rm_cmd = "$RM -r $TMP_DIR"; + safesystem("$rm_cmd") or die("ERROR: could not run:\n $rm_cmd"); +} + +# utility functions + +sub safesystem { + print STDERR "Executing: @_\n"; + system(@_); + if ($? == -1) { + print STDERR "ERROR: Failed to execute: @_\n $!\n"; + exit(1); + } + elsif ($? & 127) { + printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", + ($? & 127), ($? & 128) ? 'with' : 'without'; + exit(1); + } + else { + my $exitcode = $? >> 8; + print STDERR "Exit code: $exitcode\n" if $exitcode; + return ! $exitcode; + } +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} + +sub get_moses_ini_params { + + open(MOSES_READER, $moses_ini); + while() { + my($line) = $_; + chomp($line); + + if($line eq "[ttable-file]"){ + $tableLine = ; + chomp($tableLine); + ($_,$_,$_,$_,$TRANSLATION_TABLE_FILE) = split(" ",$tableLine); # put the other parameters there if needed + } + if($line eq "[distortion-file]"){ + $tableLine = ; + chomp($tableLine); + ($_,$_,$_,$REORDERING_TABLE_FILE) = split(" ",$tableLine); # put the other parameters there if needed + } + } + close(MOSES_READER); +} + +sub get_number_of_phrases { + my $ret = 0; + open(TABLE_READER, &open_compressed($TRANSLATION_TABLE_FILE)) or die "ERROR: Can't read $TRANSLATION_TABLE_FILE"; + + while() { + $ret++; + } + + close (TABLE_READER); + return $ret; +} + +sub split_file_into_chunks { + my ($file_to_split, $chunk_size, $number_of_phrases_to_process) = @_; + open(SOURCE_READER, &open_compressed($file_to_split)) or die "ERROR: Can't read $file_to_split"; + my $FRAG_SOURCE_WRITER; + for(my $i = 0; $i < $number_of_phrases_to_process && !eof(SOURCE_READER); $i++) { + if(($i % $chunk_size) == 0){ # open fragmented file to write + my $frag_file = &get_fragmented_file_name($file_to_split, $i, $chunk_size); + open(FRAG_SOURCE_WRITER, ">".$frag_file) or die "ERROR: Can't write $frag_file"; + } + my $line = ; + print FRAG_SOURCE_WRITER $line; + if((%i % $chunk_size) == $chunk_size - 1 || (%i % $chunk_size) == $number_of_phrases_to_process - 1){ # close fragmented file before opening a new one + close(FRAG_SOURCE_WRITER); + } + } +} + + diff --git a/contrib/relent-filter/scripts/interpolateScores.pl b/contrib/relent-filter/scripts/interpolateScores.pl new file mode 100644 index 000000000..b204e951a --- /dev/null +++ b/contrib/relent-filter/scripts/interpolateScores.pl @@ -0,0 +1,94 @@ +#!/usr/bin/perl -w +use Getopt::Long; +use File::Basename; +use POSIX; + +$operation="+"; + +# read arguments +$_HELP = 1 if (@ARGV < 1 or !GetOptions ("files=s" => \$files, #moses conf file +"weights=s" => \$weights, +"operation=s" => \$operation)); #directory to put all the output files + + +# help message if arguments are not correct +if ($_HELP) { + print "Relative Entropy Pruning +Usage: perl interpolateScores.pl [PARAMS] +Function: interpolates any number of score files interlated by their weights +Authors: Wang Ling ( lingwang at cs dot cmu dot edu ) +PARAMS: + -files=s : table files to interpolate separated by a space (Ex \"file1 file2 file3\") + -weights : interpolation weights separated by a space (Ex \"0.3 0.3 0.4\") + -operation : +,* or min depending on the operation to perform to combine scores +For any questions contact lingwang at cs dot cmu dot edu +"; + exit(1); +} + +@FILES = split(/\s+/, $files); +@WEIGHTS = split(/\s+/, $weights); + +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; + +&interpolate(); + +sub interpolate { + my @READERS; + for($i = 0; $i < @FILES; $i++){ + local *FILE; + open(FILE, &open_compressed($FILES[$i])) or die "ERROR: Can't read $FILES[$i]"; + push(@READERS, *FILE); + } + $FIRST = $READERS[0]; + while(!eof($FIRST)) { + if($operation eq "+"){ + my $score = 0; + for($i = 0; $i < @FILES; $i++){ + my $READER = $READERS[$i]; + my $line = <$READER>; + chomp($line); + $score += $line*$WEIGHTS[$i]; + } + print "$score\n"; + } + if($operation eq "*"){ + my $score = 1; + for($i = 0; $i < @FILES; $i++){ + my $READER = $READERS[$i]; + my $line = <$READER>; + chomp($line); + $score *= $line ** $WEIGHTS[$i]; + } + print "$score\n" + } + if($operation eq "min"){ + my $score = 99999; + for($i = 0; $i < @FILES; $i++){ + my $READER = $READERS[$i]; + my $line = <$READER>; + chomp($line); + if ($score > $line*$WEIGHTS[$i]){ + $score = $line*$WEIGHTS[$i]; + } + } + print "$score\n" + + } + } +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} diff --git a/contrib/relent-filter/scripts/prunePT.pl b/contrib/relent-filter/scripts/prunePT.pl new file mode 100755 index 000000000..37dc30bad --- /dev/null +++ b/contrib/relent-filter/scripts/prunePT.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w + +# read arguments +my $tmp_dir = ""; +my $percentage = -1; +my $threshold = -1; +use Getopt::Long; +$_HELP = 1 if (@ARGV < 1 or !GetOptions ("table=s" => \$table, #table to filter +"scores=s" => \$scores_file, #scores of each phrase pair, should have same size as the table to filter +"percentage=i" => \$percentage, # percentage of phrase table to remain +"threshold=i" => \$threshold)); # threshold (score < threshold equals prune entry) + +# help message if arguments are not correct +if ($_HELP) { + print "Relative Entropy Pruning +Usage: perl prunePT.pl [PARAMS] +Function: prunes a phrase table given a score file +Authors: Wang Ling ( lingwang at cs dot cmu dot edu ) +PARAMS: + -table : table to prune + -percentage : percentage of phrase table to remain (if the scores do not allow the exact percentage if multiple entries have the same threshold, the script chooses to retain more than the given percentage) + -threshold : threshold to prune (score < threshold equals prune entry), do not use this if percentage is specified +For any questions contact lingwang at cs dot cmu dot edu +"; + exit(1); +} + + +my $THRESHOLD = $threshold; +if ($percentage != -1){ + $THRESHOLD = &get_threshold_by_percentage($percentage); +} + +my $ZCAT = "gzip -cd"; +my $BZCAT = "bzcat"; + +&prune_by_threshold($THRESHOLD); + +sub prune_by_threshold { + my $th = $_[0]; + print STDERR "pruning using threshold $th \n"; + open (SCORE_READER, &open_compressed($scores_file)); + open (TABLE_READER, &open_compressed($table)); + $number_of_phrases=0; + $number_of_unpruned_phrases=0; + while(!eof(SCORE_READER) && !eof(TABLE_READER)){ + $score_line = ; + $table_line = ; + chomp($score_line); + if($score_line >= $th){ + print $table_line; + $number_of_unpruned_phrases++; + } + $number_of_phrases++; + } + print STDERR "pruned ".($number_of_phrases - $number_of_unpruned_phrases)." phrase pairs out of $number_of_phrases\n"; +} + +sub get_threshold_by_percentage { + my $percentage = $_[0]; + $ret = 0; + + $number_of_phrases = &get_number_of_phrases(); + $stop_phrase = ($percentage * $number_of_phrases) / 100; + $phrase_number = 0; + + + open (SCORE_READER, &open_compressed($scores_file)); + while() { + my $line = $_; + + } + close (SCORE_READER); + + open (SCORE_READER, "cat $scores_file | LC_ALL=c sort -g |"); + while() { + my $line = $_; + if($phrase_number >= $stop_phrase){ + chomp($line); + $ret = $line; + last; + } + $phrase_number++; + } + + close (SCORE_READER); + return $ret; +} + +sub get_number_of_phrases { + $ret = 0; + open (SCORE_READER, $scores_file); + + while() { + $ret++; + } + + close (SCORE_READER); + return $ret; +} + +sub open_compressed { + my ($file) = @_; + print STDERR "FILE: $file\n"; + + # add extensions, if necessary + $file = $file.".bz2" if ! -e $file && -e $file.".bz2"; + $file = $file.".gz" if ! -e $file && -e $file.".gz"; + + # pipe zipped, if necessary + return "$BZCAT $file|" if $file =~ /\.bz2$/; + return "$ZCAT $file|" if $file =~ /\.gz$/; + return $file; +} diff --git a/contrib/relent-filter/sigtest-filter/Makefile b/contrib/relent-filter/sigtest-filter/Makefile new file mode 100755 index 000000000..71de9c45f --- /dev/null +++ b/contrib/relent-filter/sigtest-filter/Makefile @@ -0,0 +1,10 @@ +SALMDIR=/Users/hieuhoang/workspace/salm +FLAVOR?=o64 +INC=-I$(SALMDIR)/Src/Shared -I$(SALMDIR)/Src/SuffixArrayApplications -I$(SALMDIR)/Src/SuffixArrayApplications/SuffixArraySearch +OBJS=$(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArrayApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArraySearchApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_String.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_IDVocabulary.$(FLAVOR) + +all: filter-pt + +filter-pt: filter-pt.cpp + ./check-install $(SALMDIR) + $(CXX) -O6 $(INC) $(OBJS) -o filter-pt filter-pt.cpp diff --git a/contrib/relent-filter/sigtest-filter/README.txt b/contrib/relent-filter/sigtest-filter/README.txt new file mode 100755 index 000000000..b21129b89 --- /dev/null +++ b/contrib/relent-filter/sigtest-filter/README.txt @@ -0,0 +1,42 @@ +Re-implementation of Johnson et al. (2007)'s phrasetable filtering strategy. + +This implementation relies on Joy Zhang's SALM Suffix Array toolkit. It is +available here: + + http://projectile.sv.cmu.edu/research/public/tools/salm/salm.htm + +--Chris Dyer + +BUILD INSTRUCTIONS +--------------------------------- + +1. Download and build SALM. + +2. make SALMDIR=/path/to/SALM + + +USAGE INSTRUCTIONS +--------------------------------- + +1. Using the SALM/Bin/Linux/Index/IndexSA.O32, create a suffix array index + of the source and target sides of your training bitext. + +2. cat phrase-table.txt | ./filter-pt -e TARG.suffix -f SOURCE.suffix \ + -l + + FILTER-VALUE is the -log prob threshold described in Johnson et al. + (2007)'s paper. It may be either 'a+e', 'a-e', or a positive real + value. 'a+e' is a good setting- it filters out <1,1,1> phrase pairs. + I also recommend using -n 30, which filteres out all but the top + 30 phrase pairs, sorted by P(e|f). This was used in the paper. + +3. Run with no options to see more use-cases. + + +REFERENCES +--------------------------------- + +H. Johnson, J. Martin, G. Foster and R. Kuhn. (2007) Improving Translation + Quality by Discarding Most of the Phrasetable. In Proceedings of the 2007 + Joint Conference on Empirical Methods in Natural Language Processing and + Computational Natural Language Learning (EMNLP-CoNLL), pp. 967-975. diff --git a/contrib/relent-filter/sigtest-filter/WIN32_functions.cpp b/contrib/relent-filter/sigtest-filter/WIN32_functions.cpp new file mode 100755 index 000000000..60ddd340c --- /dev/null +++ b/contrib/relent-filter/sigtest-filter/WIN32_functions.cpp @@ -0,0 +1,231 @@ +// XGetopt.cpp Version 1.2 +// +// Author: Hans Dietrich +// hdietrich2@hotmail.com +// +// Description: +// XGetopt.cpp implements getopt(), a function to parse command lines. +// +// History +// Version 1.2 - 2003 May 17 +// - Added Unicode support +// +// Version 1.1 - 2002 March 10 +// - Added example to XGetopt.cpp module header +// +// This software is released into the public domain. +// You are free to use it in any way you like. +// +// This software is provided "as is" with no expressed +// or implied warranty. I accept no liability for any +// damage or loss of business that this software may cause. +// +/////////////////////////////////////////////////////////////////////////////// + + +/////////////////////////////////////////////////////////////////////////////// +// if you are using precompiled headers then include this line: +/////////////////////////////////////////////////////////////////////////////// + + +/////////////////////////////////////////////////////////////////////////////// +// if you are not using precompiled headers then include these lines: +//#include +//#include +//#include +/////////////////////////////////////////////////////////////////////////////// + + +#include +#include +#include +#include "WIN32_functions.h" + + +/////////////////////////////////////////////////////////////////////////////// +// +// X G e t o p t . c p p +// +// +// NAME +// getopt -- parse command line options +// +// SYNOPSIS +// int getopt(int argc, char *argv[], char *optstring) +// +// extern char *optarg; +// extern int optind; +// +// DESCRIPTION +// The getopt() function parses the command line arguments. Its +// arguments argc and argv are the argument count and array as +// passed into the application on program invocation. In the case +// of Visual C++ programs, argc and argv are available via the +// variables __argc and __argv (double underscores), respectively. +// getopt returns the next option letter in argv that matches a +// letter in optstring. (Note: Unicode programs should use +// __targv instead of __argv. Also, all character and string +// literals should be enclosed in ( ) ). +// +// optstring is a string of recognized option letters; if a letter +// is followed by a colon, the option is expected to have an argument +// that may or may not be separated from it by white space. optarg +// is set to point to the start of the option argument on return from +// getopt. +// +// Option letters may be combined, e.g., "-ab" is equivalent to +// "-a -b". Option letters are case sensitive. +// +// getopt places in the external variable optind the argv index +// of the next argument to be processed. optind is initialized +// to 0 before the first call to getopt. +// +// When all options have been processed (i.e., up to the first +// non-option argument), getopt returns EOF, optarg will point +// to the argument, and optind will be set to the argv index of +// the argument. If there are no non-option arguments, optarg +// will be set to NULL. +// +// The special option "--" may be used to delimit the end of the +// options; EOF will be returned, and "--" (and everything after it) +// will be skipped. +// +// RETURN VALUE +// For option letters contained in the string optstring, getopt +// will return the option letter. getopt returns a question mark (?) +// when it encounters an option letter not included in optstring. +// EOF is returned when processing is finished. +// +// BUGS +// 1) Long options are not supported. +// 2) The GNU double-colon extension is not supported. +// 3) The environment variable POSIXLY_CORRECT is not supported. +// 4) The + syntax is not supported. +// 5) The automatic permutation of arguments is not supported. +// 6) This implementation of getopt() returns EOF if an error is +// encountered, instead of -1 as the latest standard requires. +// +// EXAMPLE +// BOOL CMyApp::ProcessCommandLine(int argc, char *argv[]) +// { +// int c; +// +// while ((c = getopt(argc, argv, ("aBn:"))) != EOF) +// { +// switch (c) +// { +// case ('a'): +// TRACE(("option a\n")); +// // +// // set some flag here +// // +// break; +// +// case ('B'): +// TRACE( ("option B\n")); +// // +// // set some other flag here +// // +// break; +// +// case ('n'): +// TRACE(("option n: value=%d\n"), atoi(optarg)); +// // +// // do something with value here +// // +// break; +// +// case ('?'): +// TRACE(("ERROR: illegal option %s\n"), argv[optind-1]); +// return FALSE; +// break; +// +// default: +// TRACE(("WARNING: no handler for option %c\n"), c); +// return FALSE; +// break; +// } +// } +// // +// // check for non-option args here +// // +// return TRUE; +// } +// +/////////////////////////////////////////////////////////////////////////////// + +char *optarg; // global argument pointer +int optind = 0; // global argv index + +int getopt(int argc, char *argv[], char *optstring) +{ + static char *next = NULL; + if (optind == 0) + next = NULL; + + optarg = NULL; + + if (next == NULL || *next =='\0') { + if (optind == 0) + optind++; + + if (optind >= argc || argv[optind][0] != ('-') || argv[optind][1] == ('\0')) { + optarg = NULL; + if (optind < argc) + optarg = argv[optind]; + return EOF; + } + + if (strcmp(argv[optind], "--") == 0) { + optind++; + optarg = NULL; + if (optind < argc) + optarg = argv[optind]; + return EOF; + } + + next = argv[optind]; + next++; // skip past - + optind++; + } + + char c = *next++; + char *cp = strchr(optstring, c); + + if (cp == NULL || c == (':')) + return ('?'); + + cp++; + if (*cp == (':')) { + if (*next != ('\0')) { + optarg = next; + next = NULL; + } else if (optind < argc) { + optarg = argv[optind]; + optind++; + } else { + return ('?'); + } + } + + return c; +} + +// for an overview, see +// W. Press, S. Teukolsky and W. Vetterling. (1992) Numerical Recipes in C. Chapter 6.1. +double lgamma(int x) +{ + // size_t xx=(size_t)x; xx--; size_t sum=1; while (xx) { sum *= xx--; } return log((double)(sum)); + if (x <= 2) { + return 0.0; + } + static double coefs[6] = {76.18009172947146, -86.50532032941677, 24.01409824083091, -1.231739572450155, 0.1208650973866179e-2, -0.5395239384953e-5}; + double tmp=(double)x+5.5; + tmp -= (((double)x)+0.5)*log(tmp); + double y=(double)x; + double sum = 1.000000000190015; + for (size_t j=0; j<6; ++j) { + sum += coefs[j]/++y; + } + return -tmp+log(2.5066282746310005*sum/(double)x); +} \ No newline at end of file diff --git a/contrib/relent-filter/sigtest-filter/WIN32_functions.h b/contrib/relent-filter/sigtest-filter/WIN32_functions.h new file mode 100755 index 000000000..6a719392e --- /dev/null +++ b/contrib/relent-filter/sigtest-filter/WIN32_functions.h @@ -0,0 +1,24 @@ +// XGetopt.h Version 1.2 +// +// Author: Hans Dietrich +// hdietrich2@hotmail.com +// +// This software is released into the public domain. +// You are free to use it in any way you like. +// +// This software is provided "as is" with no expressed +// or implied warranty. I accept no liability for any +// damage or loss of business that this software may cause. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef XGETOPT_H +#define XGETOPT_H + +extern int optind, opterr; +extern char *optarg; + +int getopt(int argc, char *argv[], char *optstring); +double lgamma(int x); + +#endif //XGETOPT_H diff --git a/contrib/relent-filter/sigtest-filter/check-install b/contrib/relent-filter/sigtest-filter/check-install new file mode 100755 index 000000000..ba4f431e0 --- /dev/null +++ b/contrib/relent-filter/sigtest-filter/check-install @@ -0,0 +1,5 @@ +#!/usr/bin/perl -w +use strict; +my $path = shift @ARGV; +die "Can't find SALM installation path: $path\nPlease use:\n\n make SALMDIR=/path/to/SALM\n\n" unless (-d $path); +exit 0; diff --git a/contrib/relent-filter/sigtest-filter/filter-pt.cpp b/contrib/relent-filter/sigtest-filter/filter-pt.cpp new file mode 100755 index 000000000..4a51953ea --- /dev/null +++ b/contrib/relent-filter/sigtest-filter/filter-pt.cpp @@ -0,0 +1,377 @@ + +#include +#include +#include +#include +#include + +#include "_SuffixArraySearchApplicationBase.h" + +#include +#include +#include + +#ifdef WIN32 +#include "WIN32_functions.h" +#else +#include +#endif + +typedef std::set SentIdSet; +typedef std::map PhraseSetMap; + +#undef min + +// constants +const size_t MINIMUM_SIZE_TO_KEEP = 10000; // reduce this to improve memory usage, +// increase for speed +const std::string SEPARATOR = " ||| "; + +const double ALPHA_PLUS_EPS = -1000.0; // dummy value +const double ALPHA_MINUS_EPS = -2000.0; // dummy value + +// configuration params +int pfe_filter_limit = 0; // 0 = don't filter anything based on P(f|e) +bool print_cooc_counts = false; // add cooc counts to phrase table? +bool print_neglog_significance = false; // add -log(p) to phrase table? +double sig_filter_limit = 0; // keep phrase pairs with -log(sig) > sig_filter_limit +// higher = filter-more +bool pef_filter_only = false; // only filter based on pef + +// globals +PhraseSetMap esets; +double p_111 = 0.0; // alpha +size_t nremoved_sigfilter = 0; +size_t nremoved_pfefilter = 0; + +C_SuffixArraySearchApplicationBase e_sa; +C_SuffixArraySearchApplicationBase f_sa; +int num_lines; + +void usage() +{ + std::cerr << "\nFilter phrase table using significance testing as described\n" + << "in H. Johnson, et al. (2007) Improving Translation Quality\n" + << "by Discarding Most of the Phrasetable. EMNLP 2007.\n" + << "\nUsage:\n" + << "\n filter-pt -e english.suf-arr -f french.suf-arr\n" + << " [-c] [-p] [-l threshold] [-n num] < PHRASE-TABLE > FILTERED-PHRASE-TABLE\n\n" + << " [-l threshold] >0.0, a+e, or a-e: keep values that have a -log significance > this\n" + << " [-n num ] 0, 1...: 0=no filtering, >0 sort by P(e|f) and keep the top num elements\n" + << " [-c ] add the cooccurence counts to the phrase table\n" + << " [-p ] add -log(significance) to the phrasetable\n\n"; + exit(1); +} + +struct PTEntry { + PTEntry(const std::string& str, int index); + std::string f_phrase; + std::string e_phrase; + std::string extra; + std::string scores; + float pfe; + int cf; + int ce; + int cfe; + float nlog_pte; + void set_cooc_stats(int _cef, int _cf, int _ce, float nlp) { + cfe = _cef; + cf = _cf; + ce = _ce; + nlog_pte = nlp; + } + +}; + +PTEntry::PTEntry(const std::string& str, int index) : + cf(0), ce(0), cfe(0), nlog_pte(0.0) +{ + size_t pos = 0; + std::string::size_type nextPos = str.find(SEPARATOR, pos); + this->f_phrase = str.substr(pos,nextPos); + + pos = nextPos + SEPARATOR.size(); + nextPos = str.find(SEPARATOR, pos); + this->e_phrase = str.substr(pos,nextPos-pos); + + pos = nextPos + SEPARATOR.size(); + nextPos = str.find(SEPARATOR, pos); + this->scores = str.substr(pos,nextPos-pos); + + pos = nextPos + SEPARATOR.size(); + this->extra = str.substr(pos); + + int c = 0; + std::string::iterator i=scores.begin(); + if (index > 0) { + for (; i != scores.end(); ++i) { + if ((*i) == ' ') { + c++; + if (c == index) break; + } + } + } + if (i != scores.end()) { + ++i; + } + char f[24]; + char *fp=f; + while (i != scores.end() && *i != ' ') { + *fp++=*i++; + } + *fp++=0; + + this->pfe = atof(f); + + // std::cerr << "L: " << f_phrase << " ::: " << e_phrase << " ::: " << scores << " ::: " << pfe << std::endl; + // std::cerr << "X: " << extra << "\n"; +} + +struct PfeComparer { + bool operator()(const PTEntry* a, const PTEntry* b) const { + return a->pfe > b->pfe; + } +}; + +struct NlogSigThresholder { + NlogSigThresholder(float threshold) : t(threshold) {} + float t; + bool operator()(const PTEntry* a) const { + if (a->nlog_pte < t) { + delete a; + return true; + } else return false; + } +}; + +std::ostream& operator << (std::ostream& os, const PTEntry& pp) +{ + //os << pp.f_phrase << " ||| " << pp.e_phrase; + //os << " ||| " << pp.scores; + //if (pp.extra.size()>0) os << " ||| " << pp.extra; + if (print_cooc_counts) os << pp.cfe << " " << pp.cf << " " << pp.ce; + if (print_neglog_significance) os << " ||| " << pp.nlog_pte; + return os; +} + +void print(int a, int b, int c, int d, float p) +{ + std::cerr << a << "\t" << b << "\t P=" << p << "\n" + << c << "\t" << d << "\t xf=" << (double)(b)*(double)(c)/(double)(a+1)/(double)(d+1) << "\n\n"; +} + +// 2x2 (one-sided) Fisher's exact test +// see B. Moore. (2004) On Log Likelihood and the Significance of Rare Events +double fisher_exact(int cfe, int ce, int cf) +{ + assert(cfe <= ce); + assert(cfe <= cf); + + int a = cfe; + int b = (cf - cfe); + int c = (ce - cfe); + int d = (num_lines - ce - cf + cfe); + int n = a + b + c + d; + + double cp = exp(lgamma(1+a+c) + lgamma(1+b+d) + lgamma(1+a+b) + lgamma(1+c+d) - lgamma(1+n) - lgamma(1+a) - lgamma(1+b) - lgamma(1+c) - lgamma(1+d)); + double total_p = 0.0; + int tc = std::min(b,c); + for (int i=0; i<=tc; i++) { + total_p += cp; +// double lg = lgamma(1+a+c) + lgamma(1+b+d) + lgamma(1+a+b) + lgamma(1+c+d) - lgamma(1+n) - lgamma(1+a) - lgamma(1+b) - lgamma(1+c) - lgamma(1+d); double cp = exp(lg); +// print(a,b,c,d,cp); + double coef = (double)(b)*(double)(c)/(double)(a+1)/(double)(d+1); + cp *= coef; + ++a; + --c; + ++d; + --b; + } + return total_p; +} + +// input: unordered list of translation options for a single source phrase +void compute_cooc_stats_and_filter(std::vector& options) +{ + if (pfe_filter_limit>0 && options.size() > pfe_filter_limit) { + nremoved_pfefilter += (options.size() - pfe_filter_limit); + std::nth_element(options.begin(), options.begin()+pfe_filter_limit, options.end(), PfeComparer()); + for (std::vector::iterator i=options.begin()+pfe_filter_limit; i != options.end(); ++i) + delete *i; + options.erase(options.begin()+pfe_filter_limit,options.end()); + } + if (pef_filter_only) return; + + SentIdSet fset; + vector locations; + //std::cerr << "Looking up f-phrase: " << options.front()->f_phrase << "\n"; + + locations = f_sa.locateExactPhraseInCorpus(options.front()->f_phrase.c_str()); + if(locations.size()==0) { + cerr<<"No occurrences found!!\n"; + } + for (vector::iterator i=locations.begin(); + i != locations.end(); + ++i) { + fset.insert(i->sentIdInCorpus); + } + size_t cf = fset.size(); + for (std::vector::iterator i=options.begin(); i != options.end(); ++i) { + const std::string& e_phrase = (*i)->e_phrase; + size_t cef=0; + SentIdSet& eset = esets[(*i)->e_phrase]; + if (eset.empty()) { + //std::cerr << "Looking up e-phrase: " << e_phrase << "\n"; + vector locations = e_sa.locateExactPhraseInCorpus(e_phrase.c_str()); + for (vector::iterator i=locations.begin(); i!= locations.end(); ++i) { + TextLenType curSentId = i->sentIdInCorpus; + eset.insert(curSentId); + } + } + size_t ce=eset.size(); + if (ce < cf) { + for (SentIdSet::iterator i=eset.begin(); i != eset.end(); ++i) { + if (fset.find(*i) != fset.end()) cef++; + } + } else { + for (SentIdSet::iterator i=fset.begin(); i != fset.end(); ++i) { + if (eset.find(*i) != eset.end()) cef++; + } + } + double nlp = -log(fisher_exact(cef, cf, ce)); + (*i)->set_cooc_stats(cef, cf, ce, nlp); + if (ce < MINIMUM_SIZE_TO_KEEP) { + esets.erase(e_phrase); + } + } + std::vector::iterator new_end = + std::remove_if(options.begin(), options.end(), NlogSigThresholder(sig_filter_limit)); + nremoved_sigfilter += (options.end() - new_end); + options.erase(new_end,options.end()); +} + +int main(int argc, char * argv[]) +{ + int c; + const char* efile=0; + const char* ffile=0; + int pfe_index = 2; + while ((c = getopt(argc, argv, "cpf:e:i:n:l:")) != -1) { + switch (c) { + case 'e': + efile = optarg; + break; + case 'f': + ffile = optarg; + break; + case 'i': // index of pfe in phrase table + pfe_index = atoi(optarg); + break; + case 'n': // keep only the top n entries in phrase table sorted by p(f|e) (0=all) + pfe_filter_limit = atoi(optarg); + std::cerr << "P(f|e) filter limit: " << pfe_filter_limit << std::endl; + break; + case 'c': + print_cooc_counts = true; + break; + case 'p': + print_neglog_significance = true; + break; + case 'l': + std::cerr << "-l = " << optarg << "\n"; + if (strcmp(optarg,"a+e") == 0) { + sig_filter_limit = ALPHA_PLUS_EPS; + } else if (strcmp(optarg,"a-e") == 0) { + sig_filter_limit = ALPHA_MINUS_EPS; + } else { + char *x; + sig_filter_limit = strtod(optarg, &x); + } + break; + default: + usage(); + } + } + //----------------------------------------------------------------------------- + if (optind != argc || ((!efile || !ffile) && !pef_filter_only)) { + usage(); + } + + //load the indexed corpus with vocabulary(noVoc=false) and with offset(noOffset=false) + if (!pef_filter_only) { + e_sa.loadData_forSearch(efile, false, false); + f_sa.loadData_forSearch(ffile, false, false); + size_t elines = e_sa.returnTotalSentNumber(); + size_t flines = f_sa.returnTotalSentNumber(); + if (elines != flines) { + std::cerr << "Number of lines in e-corpus != number of lines in f-corpus!\n"; + usage(); + } else { + std::cerr << "Training corpus: " << elines << " lines\n"; + num_lines = elines; + } + p_111 = -log(fisher_exact(1,1,1)); + std::cerr << "\\alpha = " << p_111 << "\n"; + if (sig_filter_limit == ALPHA_MINUS_EPS) { + sig_filter_limit = p_111 - 0.001; + } else if (sig_filter_limit == ALPHA_PLUS_EPS) { + sig_filter_limit = p_111 + 0.001; + } + std::cerr << "Sig filter threshold is = " << sig_filter_limit << "\n"; + } else { + std::cerr << "Filtering using P(e|f) only. n=" << pfe_filter_limit << std::endl; + } + + char tmpString[10000]; + std::string prev = ""; + std::vector options; + size_t pt_lines = 0; + while(!cin.eof()) { + cin.getline(tmpString,10000,'\n'); + if(++pt_lines%10000==0) { + std::cerr << "."; + if(pt_lines%500000==0) std::cerr << "[n:"<0) { + PTEntry* pp = new PTEntry(tmpString, pfe_index); + if (prev != pp->f_phrase) { + prev = pp->f_phrase; + + if (!options.empty()) { // always true after first line + compute_cooc_stats_and_filter(options); + } + for (std::vector::iterator i=options.begin(); i != options.end(); ++i) { + std::cout << **i << std::endl; + delete *i; + } + options.clear(); + options.push_back(pp); + + } else { + options.push_back(pp); + } + // for(int i=0;i +#include +#include "TypeDef.h" +#include "Util.h" +#include "IOWrapper.h" +#include "Hypothesis.h" +#include "WordsRange.h" +#include "TrellisPathList.h" +#include "StaticData.h" +#include "DummyScoreProducers.h" +#include "InputFileStream.h" + +using namespace std; +using namespace Moses; + +namespace MosesCmd +{ + +IOWrapper::IOWrapper( + const vector &inputFactorOrder + , const vector &outputFactorOrder + , const FactorMask &inputFactorUsed + , size_t nBestSize + , const string &nBestFilePath) + :m_inputFactorOrder(inputFactorOrder) + ,m_outputFactorOrder(outputFactorOrder) + ,m_inputFactorUsed(inputFactorUsed) + ,m_inputFile(NULL) + ,m_inputStream(&std::cin) + ,m_nBestStream(NULL) + ,m_outputWordGraphStream(NULL) + ,m_outputSearchGraphStream(NULL) + ,m_detailedTranslationReportingStream(NULL) + ,m_alignmentOutputStream(NULL) +{ + Initialization(inputFactorOrder, outputFactorOrder + , inputFactorUsed + , nBestSize, nBestFilePath); +} + +IOWrapper::IOWrapper(const std::vector &inputFactorOrder + , const std::vector &outputFactorOrder + , const FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath + , const std::string &inputFilePath) + :m_inputFactorOrder(inputFactorOrder) + ,m_outputFactorOrder(outputFactorOrder) + ,m_inputFactorUsed(inputFactorUsed) + ,m_inputFilePath(inputFilePath) + ,m_inputFile(new InputFileStream(inputFilePath)) + ,m_nBestStream(NULL) + ,m_outputWordGraphStream(NULL) + ,m_outputSearchGraphStream(NULL) + ,m_detailedTranslationReportingStream(NULL) + ,m_alignmentOutputStream(NULL) +{ + Initialization(inputFactorOrder, outputFactorOrder + , inputFactorUsed + , nBestSize, nBestFilePath); + + m_inputStream = m_inputFile; +} + +IOWrapper::~IOWrapper() +{ + if (m_inputFile != NULL) + delete m_inputFile; + if (m_nBestStream != NULL && !m_surpressSingleBestOutput) { + // outputting n-best to file, rather than stdout. need to close file and delete obj + delete m_nBestStream; + } + if (m_outputWordGraphStream != NULL) { + delete m_outputWordGraphStream; + } + if (m_outputSearchGraphStream != NULL) { + delete m_outputSearchGraphStream; + } + delete m_detailedTranslationReportingStream; + delete m_alignmentOutputStream; +} + +void IOWrapper::Initialization(const std::vector &/*inputFactorOrder*/ + , const std::vector &/*outputFactorOrder*/ + , const FactorMask &/*inputFactorUsed*/ + , size_t nBestSize + , const std::string &nBestFilePath) +{ + const StaticData &staticData = StaticData::Instance(); + + // n-best + m_surpressSingleBestOutput = false; + + if (nBestSize > 0) { + if (nBestFilePath == "-" || nBestFilePath == "/dev/stdout") { + m_nBestStream = &std::cout; + m_surpressSingleBestOutput = true; + } else { + std::ofstream *file = new std::ofstream; + m_nBestStream = file; + file->open(nBestFilePath.c_str()); + } + } + + // wordgraph output + if (staticData.GetOutputWordGraph()) { + string fileName = staticData.GetParam("output-word-graph")[0]; + std::ofstream *file = new std::ofstream; + m_outputWordGraphStream = file; + file->open(fileName.c_str()); + } + + +// search graph output + if (staticData.GetOutputSearchGraph()) { + string fileName; + if (staticData.GetOutputSearchGraphExtended()) + fileName = staticData.GetParam("output-search-graph-extended")[0]; + else + fileName = staticData.GetParam("output-search-graph")[0]; + std::ofstream *file = new std::ofstream; + m_outputSearchGraphStream = file; + file->open(fileName.c_str()); + } + + // detailed translation reporting + if (staticData.IsDetailedTranslationReportingEnabled()) { + const std::string &path = staticData.GetDetailedTranslationReportingFilePath(); + m_detailedTranslationReportingStream = new std::ofstream(path.c_str()); + CHECK(m_detailedTranslationReportingStream->good()); + } + + // sentence alignment output + if (! staticData.GetAlignmentOutputFile().empty()) { + m_alignmentOutputStream = new ofstream(staticData.GetAlignmentOutputFile().c_str()); + CHECK(m_alignmentOutputStream->good()); + } + +} + +InputType*IOWrapper::GetInput(InputType* inputType) +{ + if(inputType->Read(*m_inputStream, m_inputFactorOrder)) { + if (long x = inputType->GetTranslationId()) { + if (x>=m_translationId) m_translationId = x+1; + } else inputType->SetTranslationId(m_translationId++); + + return inputType; + } else { + delete inputType; + return NULL; + } +} + +/*** + * print surface factor only for the given phrase + */ +void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector &outputFactorOrder, + bool reportSegmentation, bool reportAllFactors) +{ + CHECK(outputFactorOrder.size() > 0); + const Phrase& phrase = edge.GetCurrTargetPhrase(); + if (reportAllFactors == true) { + out << phrase; + } else { + size_t size = phrase.GetSize(); + for (size_t pos = 0 ; pos < size ; pos++) { + const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]); + out << *factor; + CHECK(factor); + + for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) { + const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]); + CHECK(factor); + + out << "|" << *factor; + } + out << " "; + } + } + + // trace option "-t" + if (reportSegmentation == true && phrase.GetSize() > 0) { + out << "|" << edge.GetCurrSourceWordsRange().GetStartPos() + << "-" << edge.GetCurrSourceWordsRange().GetEndPos() << "| "; + } +} + +void OutputBestSurface(std::ostream &out, const Hypothesis *hypo, const std::vector &outputFactorOrder, + bool reportSegmentation, bool reportAllFactors) +{ + if (hypo != NULL) { + // recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence + OutputBestSurface(out, hypo->GetPrevHypo(), outputFactorOrder, reportSegmentation, reportAllFactors); + OutputSurface(out, *hypo, outputFactorOrder, reportSegmentation, reportAllFactors); + } +} + +void OutputAlignment(ostream &out, const AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset) +{ + typedef std::vector< const std::pair* > AlignVec; + AlignVec alignments = ai.GetSortedAlignments(); + + AlignVec::const_iterator it; + for (it = alignments.begin(); it != alignments.end(); ++it) { + const std::pair &alignment = **it; + out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " "; + } + +} + +void OutputAlignment(ostream &out, const vector &edges) +{ + size_t targetOffset = 0; + + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + const TargetPhrase &tp = edge.GetCurrTargetPhrase(); + size_t sourceOffset = edge.GetCurrSourceWordsRange().GetStartPos(); + + OutputAlignment(out, tp.GetAlignmentInfo(), sourceOffset, targetOffset); + + targetOffset += tp.GetSize(); + } + out << std::endl; +} + +void OutputAlignment(OutputCollector* collector, size_t lineNo , const vector &edges) +{ + ostringstream out; + OutputAlignment(out, edges); + + collector->Write(lineNo,out.str()); +} + +void OutputAlignment(OutputCollector* collector, size_t lineNo , const Hypothesis *hypo) +{ + if (collector) { + std::vector edges; + const Hypothesis *currentHypo = hypo; + while (currentHypo) { + edges.push_back(currentHypo); + currentHypo = currentHypo->GetPrevHypo(); + } + + OutputAlignment(collector,lineNo, edges); + } +} + +void OutputAlignment(OutputCollector* collector, size_t lineNo , const TrellisPath &path) +{ + if (collector) { + OutputAlignment(collector,lineNo, path.GetEdges()); + } +} + +void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/, bool reportSegmentation, bool reportAllFactors, std::ostream &out) +{ + const std::vector &edges = path.GetEdges(); + + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + OutputSurface(out, edge, StaticData::Instance().GetOutputFactorOrder(), reportSegmentation, reportAllFactors); + } + out << endl; +} + +void IOWrapper::Backtrack(const Hypothesis *hypo) +{ + + if (hypo->GetPrevHypo() != NULL) { + VERBOSE(3,hypo->GetId() << " <= "); + Backtrack(hypo->GetPrevHypo()); + } +} + +void OutputBestHypo(const std::vector& mbrBestHypo, long /*translationId*/, bool /*reportSegmentation*/, bool /*reportAllFactors*/, ostream& out) +{ + + for (size_t i = 0 ; i < mbrBestHypo.size() ; i++) { + const Factor *factor = mbrBestHypo[i].GetFactor(StaticData::Instance().GetOutputFactorOrder()[0]); + CHECK(factor); + if (i>0) out << " " << *factor; + else out << *factor; + } + out << endl; +} + + +void OutputInput(std::vector& map, const Hypothesis* hypo) +{ + if (hypo->GetPrevHypo()) { + OutputInput(map, hypo->GetPrevHypo()); + map[hypo->GetCurrSourceWordsRange().GetStartPos()] = hypo->GetSourcePhrase(); + } +} + +void OutputInput(std::ostream& os, const Hypothesis* hypo) +{ + size_t len = hypo->GetInput().GetSize(); + std::vector inp_phrases(len, 0); + OutputInput(inp_phrases, hypo); + for (size_t i=0; i& outputFactorOrder, const TranslationSystem* system, long translationId, bool reportSegmentation) +{ + const StaticData &staticData = StaticData::Instance(); + bool labeledOutput = staticData.IsLabeledNBestList(); + bool reportAllFactors = staticData.GetReportAllFactorsNBest(); + bool includeAlignment = staticData.NBestIncludesAlignment(); + bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest(); + + TrellisPathList::const_iterator iter; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const TrellisPath &path = **iter; + const std::vector &edges = path.GetEdges(); + + // print the surface factor of the translation + out << translationId << " ||| "; + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + OutputSurface(out, edge, outputFactorOrder, reportSegmentation, reportAllFactors); + } + out << " |||"; + + std::string lastName = ""; + const vector& sff = system->GetStatefulFeatureFunctions(); + for( size_t i=0; iGetScoreProducerWeightShortName() ) { + lastName = sff[i]->GetScoreProducerWeightShortName(); + out << " " << lastName << ":"; + } + vector scores = path.GetScoreBreakdown().GetScoresForProducer( sff[i] ); + for (size_t j = 0; j& slf = system->GetStatelessFeatureFunctions(); + for( size_t i=0; iGetScoreProducerWeightShortName() ) { + lastName = slf[i]->GetScoreProducerWeightShortName(); + out << " " << lastName << ":"; + } + vector scores = path.GetScoreBreakdown().GetScoresForProducer( slf[i] ); + for (size_t j = 0; j& pds = system->GetPhraseDictionaries(); + if (pds.size() > 0) { + + for( size_t i=0; iGetNumInputScores(); + vector scores = path.GetScoreBreakdown().GetScoresForProducer( pds[i] ); + for (size_t j = 0; jGetScoreProducerWeightShortName(j); + out << " " << lastName << ":"; + } + } + out << " " << scores[j]; + } + } + } + + // generation + const vector& gds = system->GetGenerationDictionaries(); + if (gds.size() > 0) { + + for( size_t i=0; iGetNumInputScores(); + vector scores = path.GetScoreBreakdown().GetScoresForProducer( gds[i] ); + for (size_t j = 0; jGetScoreProducerWeightShortName(j); + out << " " << lastName << ":"; + } + } + out << " " << scores[j]; + } + } + } + + // total + out << " ||| " << path.GetTotalScore(); + + //phrase-to-phrase alignment + if (includeAlignment) { + out << " |||"; + for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + const WordsRange &sourceRange = edge.GetCurrSourceWordsRange(); + WordsRange targetRange = path.GetTargetWordsRange(edge); + out << " " << sourceRange.GetStartPos(); + if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) { + out << "-" << sourceRange.GetEndPos(); + } + out<< "=" << targetRange.GetStartPos(); + if (targetRange.GetStartPos() < targetRange.GetEndPos()) { + out<< "-" << targetRange.GetEndPos(); + } + } + } + + if (includeWordAlignment) { + out << " ||| "; + for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + const WordsRange &sourceRange = edge.GetCurrSourceWordsRange(); + WordsRange targetRange = path.GetTargetWordsRange(edge); + const int sourceOffset = sourceRange.GetStartPos(); + const int targetOffset = targetRange.GetStartPos(); + const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignmentInfo(); + + OutputAlignment(out, ai, sourceOffset, targetOffset); + + } + } + + if (StaticData::Instance().IsPathRecoveryEnabled()) { + out << "|||"; + OutputInput(out, edges[0]); + } + + out << endl; + } + + + out <& solutions,long translationId) +{ + for (vector::const_iterator si = solutions.begin(); si != solutions.end(); ++si) { + out << translationId; + out << " |||"; + const vector mbrHypo = si->GetWords(); + for (size_t i = 0 ; i < mbrHypo.size() ; i++) { + const Factor *factor = mbrHypo[i].GetFactor(StaticData::Instance().GetOutputFactorOrder()[0]); + if (i>0) out << " " << *factor; + else out << *factor; + } + out << " |||"; + out << " map: " << si->GetMapScore(); + out << " w: " << mbrHypo.size(); + const vector& ngramScores = si->GetNgramScores(); + for (size_t i = 0; i < ngramScores.size(); ++i) { + out << " " << ngramScores[i]; + } + out << " ||| " << si->GetScore(); + + out << endl; + } +} + + +void IOWrapper::OutputLatticeMBRNBestList(const vector& solutions,long translationId) +{ + OutputLatticeMBRNBest(*m_nBestStream, solutions,translationId); +} + +bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source) +{ + delete source; + switch(inputType) { + case SentenceInput: + source = ioWrapper.GetInput(new Sentence); + break; + case ConfusionNetworkInput: + source = ioWrapper.GetInput(new ConfusionNet); + break; + case WordLatticeInput: + source = ioWrapper.GetInput(new WordLattice); + break; + default: + TRACE_ERR("Unknown input type: " << inputType << "\n"); + } + return (source ? true : false); +} + + + +IOWrapper *GetIOWrapper(const StaticData &staticData) +{ + IOWrapper *ioWrapper; + const std::vector &inputFactorOrder = staticData.GetInputFactorOrder() + ,&outputFactorOrder = staticData.GetOutputFactorOrder(); + FactorMask inputFactorUsed(inputFactorOrder); + + // io + if (staticData.GetParam("input-file").size() == 1) { + VERBOSE(2,"IO from File" << endl); + string filePath = staticData.GetParam("input-file")[0]; + + ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed + , staticData.GetNBestSize() + , staticData.GetNBestFilePath() + , filePath); + } else { + VERBOSE(1,"IO from STDOUT/STDIN" << endl); + ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed + , staticData.GetNBestSize() + , staticData.GetNBestFilePath()); + } + ioWrapper->ResetTranslationId(); + + IFVERBOSE(1) + PrintUserTime("Created input-output object"); + + return ioWrapper; +} + +} + diff --git a/contrib/relent-filter/src/IOWrapper.h b/contrib/relent-filter/src/IOWrapper.h new file mode 100755 index 000000000..e44208002 --- /dev/null +++ b/contrib/relent-filter/src/IOWrapper.h @@ -0,0 +1,142 @@ +// $Id$ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (c) 2006 University of Edinburgh +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +// example file on how to use moses library + +#ifndef moses_cmd_IOWrapper_h +#define moses_cmd_IOWrapper_h + +#include +#include +#include +#include +#include "util/check.hh" + +#include "TypeDef.h" +#include "Sentence.h" +#include "FactorTypeSet.h" +#include "FactorCollection.h" +#include "Hypothesis.h" +#include "OutputCollector.h" +#include "TrellisPathList.h" +#include "InputFileStream.h" +#include "InputType.h" +#include "WordLattice.h" +#include "LatticeMBR.h" + +namespace MosesCmd +{ + +/** Helper class that holds misc variables to write data out to command line. + */ +class IOWrapper +{ +protected: + long m_translationId; + + const std::vector &m_inputFactorOrder; + const std::vector &m_outputFactorOrder; + const Moses::FactorMask &m_inputFactorUsed; + std::string m_inputFilePath; + Moses::InputFileStream *m_inputFile; + std::istream *m_inputStream; + std::ostream *m_nBestStream + ,*m_outputWordGraphStream,*m_outputSearchGraphStream; + std::ostream *m_detailedTranslationReportingStream; + std::ofstream *m_alignmentOutputStream; + bool m_surpressSingleBestOutput; + + void Initialization(const std::vector &inputFactorOrder + , const std::vector &outputFactorOrder + , const Moses::FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath); + +public: + IOWrapper(const std::vector &inputFactorOrder + , const std::vector &outputFactorOrder + , const Moses::FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath); + + IOWrapper(const std::vector &inputFactorOrder + , const std::vector &outputFactorOrder + , const Moses::FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath + , const std::string &infilePath); + ~IOWrapper(); + + Moses::InputType* GetInput(Moses::InputType *inputType); + + void OutputBestHypo(const Moses::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors); + void OutputLatticeMBRNBestList(const std::vector& solutions,long translationId); + void Backtrack(const Moses::Hypothesis *hypo); + + void ResetTranslationId() { + m_translationId = 0; + } + + std::ofstream *GetAlignmentOutputStream() { + return m_alignmentOutputStream; + } + + std::ostream &GetOutputWordGraphStream() { + return *m_outputWordGraphStream; + } + std::ostream &GetOutputSearchGraphStream() { + return *m_outputSearchGraphStream; + } + + std::ostream &GetDetailedTranslationReportingStream() { + assert (m_detailedTranslationReportingStream); + return *m_detailedTranslationReportingStream; + } +}; + +IOWrapper *GetIOWrapper(const Moses::StaticData &staticData); +bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::InputType*& source); +void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector &outputFactorOrder, bool reportSegmentation, bool reportAllFactors); +void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList, const std::vector&, + const Moses::TranslationSystem* system, long translationId, bool reportSegmentation); +void OutputLatticeMBRNBest(std::ostream& out, const std::vector& solutions,long translationId); +void OutputBestHypo(const std::vector& mbrBestHypo, long /*translationId*/, + bool reportSegmentation, bool reportAllFactors, std::ostream& out); +void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,bool reportSegmentation, bool reportAllFactors, std::ostream &out); +void OutputInput(std::ostream& os, const Moses::Hypothesis* hypo); +void OutputAlignment(Moses::OutputCollector* collector, size_t lineNo, const Moses::Hypothesis *hypo); +void OutputAlignment(Moses::OutputCollector* collector, size_t lineNo, const Moses::TrellisPath &path); + + +} + +#endif diff --git a/contrib/relent-filter/src/Jamfile b/contrib/relent-filter/src/Jamfile new file mode 100755 index 000000000..c0aa6160d --- /dev/null +++ b/contrib/relent-filter/src/Jamfile @@ -0,0 +1,6 @@ +alias deps : ../../../moses/src//moses ; + +exe calcDivergence : Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp LatticeMBR.cpp RelativeEntropyCalc.cpp deps ; + +alias programs : calcDivergence ; + diff --git a/contrib/relent-filter/src/LatticeMBR.cpp b/contrib/relent-filter/src/LatticeMBR.cpp new file mode 100755 index 000000000..2bd62747e --- /dev/null +++ b/contrib/relent-filter/src/LatticeMBR.cpp @@ -0,0 +1,669 @@ +/* + * LatticeMBR.cpp + * moses-cmd + * + * Created by Abhishek Arun on 26/01/2010. + * Copyright 2010 __MyCompanyName__. All rights reserved. + * + */ + +#include "LatticeMBR.h" +#include "StaticData.h" +#include +#include + +using namespace std; +using namespace Moses; + +namespace MosesCmd +{ + +size_t bleu_order = 4; +float UNKNGRAMLOGPROB = -20; +void GetOutputWords(const TrellisPath &path, vector &translation) +{ + const std::vector &edges = path.GetEdges(); + + // print the surface factor of the translation + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + const Phrase &phrase = edge.GetCurrTargetPhrase(); + size_t size = phrase.GetSize(); + for (size_t pos = 0 ; pos < size ; pos++) { + translation.push_back(phrase.GetWord(pos)); + } + } +} + + +void extract_ngrams(const vector& sentence, map < Phrase, int > & allngrams) +{ + for (int k = 0; k < (int)bleu_order; k++) { + for(int i =0; i < max((int)sentence.size()-k,0); i++) { + Phrase ngram( k+1); + for ( int j = i; j<= i+k; j++) { + ngram.AddWord(sentence[j]); + } + ++allngrams[ngram]; + } + } +} + + + +void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score) +{ + set::const_iterator ngramIter = m_ngrams.find(ngram); + if (ngramIter == m_ngrams.end()) { + ngramIter = m_ngrams.insert(ngram).first; + } + map& ngramScores = m_scores[node]; + map::iterator scoreIter = ngramScores.find(&(*ngramIter)); + if (scoreIter == ngramScores.end()) { + ngramScores[&(*ngramIter)] = score; + } else { + ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second); + } +} + +NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node) +{ + return m_scores[node].begin(); +} + + +NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node) +{ + return m_scores[node].end(); +} + +LatticeMBRSolution::LatticeMBRSolution(const TrellisPath& path, bool isMap) : + m_score(0.0f) +{ + const std::vector &edges = path.GetEdges(); + + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + const Phrase &phrase = edge.GetCurrTargetPhrase(); + size_t size = phrase.GetSize(); + for (size_t pos = 0 ; pos < size ; pos++) { + m_words.push_back(phrase.GetWord(pos)); + } + } + if (isMap) { + m_mapScore = path.GetTotalScore(); + } else { + m_mapScore = 0; + } +} + + +void LatticeMBRSolution::CalcScore(map& finalNgramScores, const vector& thetas, float mapWeight) +{ + m_ngramScores.assign(thetas.size()-1, -10000); + + map < Phrase, int > counts; + extract_ngrams(m_words,counts); + + //Now score this translation + m_score = thetas[0] * m_words.size(); + + //Calculate the ngramScores, working in log space at first + for (map < Phrase, int >::iterator ngrams = counts.begin(); ngrams != counts.end(); ++ngrams) { + float ngramPosterior = UNKNGRAMLOGPROB; + map::const_iterator ngramPosteriorIt = finalNgramScores.find(ngrams->first); + if (ngramPosteriorIt != finalNgramScores.end()) { + ngramPosterior = ngramPosteriorIt->second; + } + size_t ngramSize = ngrams->first.GetSize(); + m_ngramScores[ngramSize-1] = log_sum(log((float)ngrams->second) + ngramPosterior,m_ngramScores[ngramSize-1]); + } + + //convert from log to probability and create weighted sum + for (size_t i = 0; i < m_ngramScores.size(); ++i) { + m_ngramScores[i] = exp(m_ngramScores[i]); + m_score += thetas[i+1] * m_ngramScores[i]; + } + + + //The map score + m_score += m_mapScore*mapWeight; +} + + +void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set > & outgoingHyps, map >& incomingEdges, + const vector< float> & estimatedScores, const Hypothesis* bestHypo, size_t edgeDensity, float scale) +{ + + //Need hyp 0 in connectedHyp - Find empty hypothesis + VERBOSE(2,"Pruning lattice to edge density " << edgeDensity << endl); + const Hypothesis* emptyHyp = connectedHyp.at(0); + while (emptyHyp->GetId() != 0) { + emptyHyp = emptyHyp->GetPrevHypo(); + } + connectedHyp.push_back(emptyHyp); //Add it to list of hyps + + //Need hyp 0's outgoing Hyps + for (size_t i = 0; i < connectedHyp.size(); ++i) { + if (connectedHyp[i]->GetId() > 0 && connectedHyp[i]->GetPrevHypo()->GetId() == 0) + outgoingHyps[emptyHyp].insert(connectedHyp[i]); + } + + //sort hyps based on estimated scores - do so by copying to multimap + multimap sortHypsByVal; + for (size_t i =0; i < estimatedScores.size(); ++i) { + sortHypsByVal.insert(make_pair(estimatedScores[i], connectedHyp[i])); + } + + multimap::const_iterator it = --sortHypsByVal.end(); + float bestScore = it->first; + //store best score as score of hyp 0 + sortHypsByVal.insert(make_pair(bestScore, emptyHyp)); + + + IFVERBOSE(3) { + for (multimap::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) { + const Hypothesis* currHyp = it->second; + cerr << "Hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl; + } + } + + + set survivingHyps; //store hyps that make the cut in this + + VERBOSE(2, "BEST HYPO TARGET LENGTH : " << bestHypo->GetSize() << endl) + size_t numEdgesTotal = edgeDensity * bestHypo->GetSize(); //as per Shankar, aim for (density * target length of MAP solution) arcs + size_t numEdgesCreated = 0; + VERBOSE(2, "Target edge count: " << numEdgesTotal << endl); + + float prevScore = -999999; + + //now iterate over multimap + for (multimap::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) { + float currEstimatedScore = it->first; + const Hypothesis* currHyp = it->second; + + if (numEdgesCreated >= numEdgesTotal && prevScore > currEstimatedScore) //if this hyp has equal estimated score to previous, include its edges too + break; + + prevScore = currEstimatedScore; + VERBOSE(3, "Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl) + VERBOSE(3, "Considering hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl) + + survivingHyps.insert(currHyp); //CurrHyp made the cut + + // is its best predecessor already included ? + if (survivingHyps.find(currHyp->GetPrevHypo()) != survivingHyps.end()) { //yes, then add an edge + vector & edges = incomingEdges[currHyp]; + Edge winningEdge(currHyp->GetPrevHypo(),currHyp,scale*(currHyp->GetScore() - currHyp->GetPrevHypo()->GetScore()),currHyp->GetCurrTargetPhrase()); + edges.push_back(winningEdge); + ++numEdgesCreated; + } + + //let's try the arcs too + const ArcList *arcList = currHyp->GetArcList(); + if (arcList != NULL) { + ArcList::const_iterator iterArcList; + for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) { + const Hypothesis *loserHypo = *iterArcList; + const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo(); + if (survivingHyps.find(loserPrevHypo) != survivingHyps.end()) { //found it, add edge + double arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore(); + Edge losingEdge(loserPrevHypo, currHyp, arcScore*scale, loserHypo->GetCurrTargetPhrase()); + vector & edges = incomingEdges[currHyp]; + edges.push_back(losingEdge); + ++numEdgesCreated; + } + } + } + + //Now if a successor node has already been visited, add an edge connecting the two + map < const Hypothesis*, set < const Hypothesis* > >::const_iterator outgoingIt = outgoingHyps.find(currHyp); + + if (outgoingIt != outgoingHyps.end()) {//currHyp does have successors + const set & outHyps = outgoingIt->second; //the successors + for (set::const_iterator outHypIts = outHyps.begin(); outHypIts != outHyps.end(); ++outHypIts) { + const Hypothesis* succHyp = *outHypIts; + + if (survivingHyps.find(succHyp) == survivingHyps.end()) //Have we encountered the successor yet? + continue; //No, move on to next + + //Curr Hyp can be : a) the best predecessor of succ b) or an arc attached to succ + if (succHyp->GetPrevHypo() == currHyp) { //best predecessor + vector & succEdges = incomingEdges[succHyp]; + Edge succWinningEdge(currHyp, succHyp, scale*(succHyp->GetScore() - currHyp->GetScore()), succHyp->GetCurrTargetPhrase()); + succEdges.push_back(succWinningEdge); + survivingHyps.insert(succHyp); + ++numEdgesCreated; + } + + //now, let's find an arc + const ArcList *arcList = succHyp->GetArcList(); + if (arcList != NULL) { + ArcList::const_iterator iterArcList; + //QUESTION: What happens if there's more than one loserPrevHypo? + for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) { + const Hypothesis *loserHypo = *iterArcList; + const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo(); + if (loserPrevHypo == currHyp) { //found it + vector & succEdges = incomingEdges[succHyp]; + double arcScore = loserHypo->GetScore() - currHyp->GetScore(); + Edge losingEdge(currHyp, succHyp,scale* arcScore, loserHypo->GetCurrTargetPhrase()); + succEdges.push_back(losingEdge); + ++numEdgesCreated; + } + } + } + } + } + } + + connectedHyp.clear(); + for (set ::iterator it = survivingHyps.begin(); it != survivingHyps.end(); ++it) { + connectedHyp.push_back(*it); + } + + VERBOSE(2, "Done! Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl) + + IFVERBOSE(3) { + cerr << "Surviving hyps: " ; + for (set ::iterator it = survivingHyps.begin(); it != survivingHyps.end(); ++it) { + cerr << (*it)->GetId() << " "; + } + cerr << endl; + } + + +} + +void calcNgramExpectations(Lattice & connectedHyp, map >& incomingEdges, + map& finalNgramScores, bool posteriors) +{ + + sort(connectedHyp.begin(),connectedHyp.end(),ascendingCoverageCmp); //sort by increasing source word cov + + /*cerr << "Lattice:" << endl; + for (Lattice::const_iterator i = connectedHyp.begin(); i != connectedHyp.end(); ++i) { + const Hypothesis* h = *i; + cerr << *h << endl; + const vector& edges = incomingEdges[h]; + for (size_t e = 0; e < edges.size(); ++e) { + cerr << edges[e]; + } + }*/ + + map forwardScore; + forwardScore[connectedHyp[0]] = 0.0f; //forward score of hyp 0 is 1 (or 0 in logprob space) + set< const Hypothesis *> finalHyps; //store completed hyps + + NgramScores ngramScores;//ngram scores for each hyp + + for (size_t i = 1; i < connectedHyp.size(); ++i) { + const Hypothesis* currHyp = connectedHyp[i]; + if (currHyp->GetWordsBitmap().IsComplete()) { + finalHyps.insert(currHyp); + } + + VERBOSE(3, "Processing hyp: " << currHyp->GetId() << ", num words cov= " << currHyp->GetWordsBitmap().GetNumWordsCovered() << endl) + + vector & edges = incomingEdges[currHyp]; + for (size_t e = 0; e < edges.size(); ++e) { + const Edge& edge = edges[e]; + if (forwardScore.find(currHyp) == forwardScore.end()) { + forwardScore[currHyp] = forwardScore[edge.GetTailNode()] + edge.GetScore(); + VERBOSE(3, "Fwd score["<GetId()<<"] = fwdScore["<GetId() << "] + edge Score: " << edge.GetScore() << endl) + } else { + forwardScore[currHyp] = log_sum(forwardScore[currHyp], forwardScore[edge.GetTailNode()] + edge.GetScore()); + VERBOSE(3, "Fwd score["<GetId()<<"] += fwdScore["<GetId() << "] + edge Score: " << edge.GetScore() << endl) + } + } + + //Process ngrams now + for (size_t j =0 ; j < edges.size(); ++j) { + Edge& edge = edges[j]; + const NgramHistory & incomingPhrases = edge.GetNgrams(incomingEdges); + + //let's first score ngrams introduced by this edge + for (NgramHistory::const_iterator it = incomingPhrases.begin(); it != incomingPhrases.end(); ++it) { + const Phrase& ngram = it->first; + const PathCounts& pathCounts = it->second; + VERBOSE(4, "Calculating score for: " << it->first << endl) + + for (PathCounts::const_iterator pathCountIt = pathCounts.begin(); pathCountIt != pathCounts.end(); ++pathCountIt) { + //Score of an n-gram is forward score of head node of leftmost edge + all edge scores + const Path& path = pathCountIt->first; + //cerr << "path count for " << ngram << " is " << pathCountIt->second << endl; + float score = forwardScore[path[0]->GetTailNode()]; + for (size_t i = 0; i < path.size(); ++i) { + score += path[i]->GetScore(); + } + //if we're doing expectations, then the number of times the ngram + //appears on the path is relevant. + size_t count = posteriors ? 1 : pathCountIt->second; + for (size_t k = 0; k < count; ++k) { + ngramScores.addScore(currHyp,ngram,score); + } + } + } + + //Now score ngrams that are just being propagated from the history + for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode()); + it != ngramScores.nodeEnd(edge.GetTailNode()); ++it) { + const Phrase & currNgram = *(it->first); + float currNgramScore = it->second; + VERBOSE(4, "Calculating score for: " << currNgram << endl) + + // For posteriors, don't double count ngrams + if (!posteriors || incomingPhrases.find(currNgram) == incomingPhrases.end()) { + float score = edge.GetScore() + currNgramScore; + ngramScores.addScore(currHyp,currNgram,score); + } + } + + } + } + + float Z = 9999999; //the total score of the lattice + + //Done - Print out ngram posteriors for final hyps + for (set< const Hypothesis *>::iterator finalHyp = finalHyps.begin(); finalHyp != finalHyps.end(); ++finalHyp) { + const Hypothesis* hyp = *finalHyp; + + for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(hyp); it != ngramScores.nodeEnd(hyp); ++it) { + const Phrase& ngram = *(it->first); + if (finalNgramScores.find(ngram) == finalNgramScores.end()) { + finalNgramScores[ngram] = it->second; + } else { + finalNgramScores[ngram] = log_sum(it->second, finalNgramScores[ngram]); + } + } + + if (Z == 9999999) { + Z = forwardScore[hyp]; + } else { + Z = log_sum(Z, forwardScore[hyp]); + } + } + + //Z *= scale; //scale the score + + for (map::iterator finalScoresIt = finalNgramScores.begin(); finalScoresIt != finalNgramScores.end(); ++finalScoresIt) { + finalScoresIt->second = finalScoresIt->second - Z; + IFVERBOSE(2) { + VERBOSE(2,finalScoresIt->first << " [" << finalScoresIt->second << "]" << endl); + } + } + +} + +const NgramHistory& Edge::GetNgrams(map > & incomingEdges) +{ + + if (m_ngrams.size() > 0) + return m_ngrams; + + const Phrase& currPhrase = GetWords(); + //Extract the n-grams local to this edge + for (size_t start = 0; start < currPhrase.GetSize(); ++start) { + for (size_t end = start; end < start + bleu_order; ++end) { + if (end < currPhrase.GetSize()) { + Phrase edgeNgram(end-start+1); + for (size_t index = start; index <= end; ++index) { + edgeNgram.AddWord(currPhrase.GetWord(index)); + } + //cout << "Inserting Phrase : " << edgeNgram << endl; + vector edgeHistory; + edgeHistory.push_back(this); + storeNgramHistory(edgeNgram, edgeHistory); + } else { + break; + } + } + } + + map >::iterator it = incomingEdges.find(m_tailNode); + if (it != incomingEdges.end()) { //node has incoming edges + vector & inEdges = it->second; + + for (vector::iterator edge = inEdges.begin(); edge != inEdges.end(); ++edge) {//add the ngrams straddling prev and curr edge + const NgramHistory & edgeIncomingNgrams = edge->GetNgrams(incomingEdges); + for (NgramHistory::const_iterator edgeInNgramHist = edgeIncomingNgrams.begin(); edgeInNgramHist != edgeIncomingNgrams.end(); ++edgeInNgramHist) { + const Phrase& edgeIncomingNgram = edgeInNgramHist->first; + const PathCounts & edgeIncomingNgramPaths = edgeInNgramHist->second; + size_t back = min(edgeIncomingNgram.GetSize(), edge->GetWordsSize()); + const Phrase& edgeWords = edge->GetWords(); + IFVERBOSE(3) { + cerr << "Edge: "<< *edge <first; + newNgramPath.push_back(this); + storeNgramHistory(newNgram, newNgramPath, pathIt->second); + } + } + } + } + } + } + return m_ngrams; +} + +//Add the last lastN words of origPhrase to targetPhrase +void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const +{ + size_t origSize = origPhrase.GetSize(); + size_t startIndex = origSize - lastN; + for (size_t index = startIndex; index < origPhrase.GetSize(); ++index) { + targetPhrase.AddWord(origPhrase.GetWord(index)); + } +} + +bool Edge::operator< (const Edge& compare ) const +{ + if (m_headNode->GetId() < compare.m_headNode->GetId()) + return true; + if (compare.m_headNode->GetId() < m_headNode->GetId()) + return false; + if (m_tailNode->GetId() < compare.m_tailNode->GetId()) + return true; + if (compare.m_tailNode->GetId() < m_tailNode->GetId()) + return false; + return GetScore() < compare.GetScore(); +} + +ostream& operator<< (ostream& out, const Edge& edge) +{ + out << "Head: " << edge.m_headNode->GetId() << ", Tail: " << edge.m_tailNode->GetId() << ", Score: " << edge.m_score << ", Phrase: " << edge.m_targetPhrase << endl; + return out; +} + +bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b) +{ + return a->GetWordsBitmap().GetNumWordsCovered() < b->GetWordsBitmap().GetNumWordsCovered(); +} + +void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList, + vector& solutions, size_t n) +{ + const StaticData& staticData = StaticData::Instance(); + std::map < int, bool > connected; + std::vector< const Hypothesis *> connectedList; + map ngramPosteriors; + std::map < const Hypothesis*, set > outgoingHyps; + map > incomingEdges; + vector< float> estimatedScores; + manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores); + pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale()); + calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true); + + vector mbrThetas = staticData.GetLatticeMBRThetas(); + float p = staticData.GetLatticeMBRPrecision(); + float r = staticData.GetLatticeMBRPRatio(); + float mapWeight = staticData.GetLatticeMBRMapWeight(); + if (mbrThetas.size() == 0) { //thetas not specified on the command line, use p and r instead + mbrThetas.push_back(-1); //Theta 0 + mbrThetas.push_back(1/(bleu_order*p)); + for (size_t i = 2; i <= bleu_order; ++i) { + mbrThetas.push_back(mbrThetas[i-1] / r); + } + } + IFVERBOSE(2) { + VERBOSE(2,"Thetas: "); + for (size_t i = 0; i < mbrThetas.size(); ++i) { + VERBOSE(2,mbrThetas[i] << " "); + } + VERBOSE(2,endl); + } + TrellisPathList::const_iterator iter; + size_t ctr = 0; + LatticeMBRSolutionComparator comparator; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter, ++ctr) { + const TrellisPath &path = **iter; + solutions.push_back(LatticeMBRSolution(path,iter==nBestList.begin())); + solutions.back().CalcScore(ngramPosteriors,mbrThetas,mapWeight); + sort(solutions.begin(), solutions.end(), comparator); + while (solutions.size() > n) { + solutions.pop_back(); + } + } + VERBOSE(2,"LMBR Score: " << solutions[0].GetScore() << endl); +} + +vector doLatticeMBR(Manager& manager, TrellisPathList& nBestList) +{ + + vector solutions; + getLatticeMBRNBest(manager, nBestList, solutions,1); + return solutions.at(0).GetWords(); +} + +const TrellisPath doConsensusDecoding(Manager& manager, TrellisPathList& nBestList) +{ + static const int BLEU_ORDER = 4; + static const float SMOOTH = 1; + + //calculate the ngram expectations + const StaticData& staticData = StaticData::Instance(); + std::map < int, bool > connected; + std::vector< const Hypothesis *> connectedList; + map ngramExpectations; + std::map < const Hypothesis*, set > outgoingHyps; + map > incomingEdges; + vector< float> estimatedScores; + manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores); + pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale()); + calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false); + + //expected length is sum of expected unigram counts + //cerr << "Thread " << pthread_self() << " Ngram expectations size: " << ngramExpectations.size() << endl; + float ref_length = 0.0f; + for (map::const_iterator ref_iter = ngramExpectations.begin(); + ref_iter != ngramExpectations.end(); ++ref_iter) { + //cerr << "Ngram: " << ref_iter->first << " score: " << + // ref_iter->second << endl; + if (ref_iter->first.GetSize() == 1) { + ref_length += exp(ref_iter->second); + // cerr << "Expected for " << ref_iter->first << " is " << exp(ref_iter->second) << endl; + } + } + + VERBOSE(2,"REF Length: " << ref_length << endl); + + //use the ngram expectations to rescore the nbest list. + TrellisPathList::const_iterator iter; + TrellisPathList::const_iterator best = nBestList.end(); + float bestScore = -100000; + //cerr << "nbest list size: " << nBestList.GetSize() << endl; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const TrellisPath &path = **iter; + vector words; + map ngrams; + GetOutputWords(path,words); + /*for (size_t i = 0; i < words.size(); ++i) { + cerr << words[i].GetFactor(0)->GetString() << " "; + } + cerr << endl; + */ + extract_ngrams(words,ngrams); + + vector comps(2*BLEU_ORDER+1); + float logbleu = 0.0; + float brevity = 0.0; + int hyp_length = words.size(); + for (int i = 0; i < BLEU_ORDER; ++i) { + comps[2*i] = 0.0; + comps[2*i+1] = max(hyp_length-i,0); + } + + for (map::const_iterator hyp_iter = ngrams.begin(); + hyp_iter != ngrams.end(); ++hyp_iter) { + map::const_iterator ref_iter = ngramExpectations.find(hyp_iter->first); + if (ref_iter != ngramExpectations.end()) { + comps[2*(hyp_iter->first.GetSize()-1)] += min(exp(ref_iter->second), (float)(hyp_iter->second)); + } + + } + comps[comps.size()-1] = ref_length; + /*for (size_t i = 0; i < comps.size(); ++i) { + cerr << comps[i] << " "; + } + cerr << endl; + */ + + float score = 0.0f; + if (comps[0] != 0) { + for (int i=0; i 0 ) { + logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH); + } else { + logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]); + } + } + logbleu /= BLEU_ORDER; + brevity = 1.0-(float)comps[comps.size()-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length + if (brevity < 0.0) { + logbleu += brevity; + } + score = exp(logbleu); + } + + //cerr << "score: " << score << " bestScore: " << bestScore << endl; + if (score > bestScore) { + bestScore = score; + best = iter; + VERBOSE(2,"NEW BEST: " << score << endl); + //for (size_t i = 0; i < comps.size(); ++i) { + // cerr << comps[i] << " "; + //} + //cerr << endl; + } + } + + assert (best != nBestList.end()); + return **best; + //vector bestWords; + //GetOutputWords(**best,bestWords); + //return bestWords; +} + +} + + diff --git a/contrib/relent-filter/src/LatticeMBR.h b/contrib/relent-filter/src/LatticeMBR.h new file mode 100755 index 000000000..14a2e22da --- /dev/null +++ b/contrib/relent-filter/src/LatticeMBR.h @@ -0,0 +1,153 @@ +/* + * LatticeMBR.h + * moses-cmd + * + * Created by Abhishek Arun on 26/01/2010. + * Copyright 2010 __MyCompanyName__. All rights reserved. + * + */ + +#ifndef moses_cmd_LatticeMBR_h +#define moses_cmd_LatticeMBR_h + +#include +#include +#include +#include "Hypothesis.h" +#include "Manager.h" +#include "TrellisPathList.h" + + + +namespace MosesCmd +{ + +class Edge; + +typedef std::vector< const Moses::Hypothesis *> Lattice; +typedef std::vector Path; +typedef std::map PathCounts; +typedef std::map NgramHistory; + +class Edge +{ + const Moses::Hypothesis* m_tailNode; + const Moses::Hypothesis* m_headNode; + float m_score; + Moses::TargetPhrase m_targetPhrase; + NgramHistory m_ngrams; + +public: + Edge(const Moses::Hypothesis* from, const Moses::Hypothesis* to, float score, const Moses::TargetPhrase& targetPhrase) : m_tailNode(from), m_headNode(to), m_score(score), m_targetPhrase(targetPhrase) { + //cout << "Creating new edge from Node " << from->GetId() << ", to Node : " << to->GetId() << ", score: " << score << " phrase: " << targetPhrase << endl; + } + + const Moses::Hypothesis* GetHeadNode() const { + return m_headNode; + } + + const Moses::Hypothesis* GetTailNode() const { + return m_tailNode; + } + + float GetScore() const { + return m_score; + } + + size_t GetWordsSize() const { + return m_targetPhrase.GetSize(); + } + + const Moses::Phrase& GetWords() const { + return m_targetPhrase; + } + + friend std::ostream& operator<< (std::ostream& out, const Edge& edge); + + const NgramHistory& GetNgrams( std::map > & incomingEdges) ; + + bool operator < (const Edge & compare) const; + + void GetPhraseSuffix(const Moses::Phrase& origPhrase, size_t lastN, Moses::Phrase& targetPhrase) const; + + void storeNgramHistory(const Moses::Phrase& phrase, Path & path, size_t count = 1) { + m_ngrams[phrase][path]+= count; + } + +}; + +/** +* Data structure to hold the ngram scores as we traverse the lattice. Maps (hypo,ngram) to score +*/ +class NgramScores +{ +public: + NgramScores() {} + + /** logsum this score to the existing score */ + void addScore(const Moses::Hypothesis* node, const Moses::Phrase& ngram, float score); + + /** Iterate through ngrams for selected node */ + typedef std::map::const_iterator NodeScoreIterator; + NodeScoreIterator nodeBegin(const Moses::Hypothesis* node); + NodeScoreIterator nodeEnd(const Moses::Hypothesis* node); + +private: + std::set m_ngrams; + std::map > m_scores; +}; + + +/** Holds a lattice mbr solution, and its scores */ +class LatticeMBRSolution +{ +public: + /** Read the words from the path */ + LatticeMBRSolution(const Moses::TrellisPath& path, bool isMap); + const std::vector& GetNgramScores() const { + return m_ngramScores; + } + const std::vector& GetWords() const { + return m_words; + } + float GetMapScore() const { + return m_mapScore; + } + float GetScore() const { + return m_score; + } + + /** Initialise ngram scores */ + void CalcScore(std::map& finalNgramScores, const std::vector& thetas, float mapWeight); + +private: + std::vector m_words; + float m_mapScore; + std::vector m_ngramScores; + float m_score; +}; + +struct LatticeMBRSolutionComparator { + bool operator()(const LatticeMBRSolution& a, const LatticeMBRSolution& b) { + return a.GetScore() > b.GetScore(); + } +}; + +void pruneLatticeFB(Lattice & connectedHyp, std::map < const Moses::Hypothesis*, std::set > & outgoingHyps, std::map >& incomingEdges, + const std::vector< float> & estimatedScores, const Moses::Hypothesis*, size_t edgeDensity,float scale); + +//Use the ngram scores to rerank the nbest list, return at most n solutions +void getLatticeMBRNBest(Moses::Manager& manager, Moses::TrellisPathList& nBestList, std::vector& solutions, size_t n); +//calculate expectated ngram counts, clipping at 1 (ie calculating posteriors) if posteriors==true. +void calcNgramExpectations(Lattice & connectedHyp, std::map >& incomingEdges, std::map& finalNgramScores, bool posteriors); +void GetOutputFactors(const Moses::TrellisPath &path, std::vector &translation); +void extract_ngrams(const std::vector& sentence, std::map < Moses::Phrase, int > & allngrams); +bool ascendingCoverageCmp(const Moses::Hypothesis* a, const Moses::Hypothesis* b); +std::vector doLatticeMBR(Moses::Manager& manager, Moses::TrellisPathList& nBestList); +const Moses::TrellisPath doConsensusDecoding(Moses::Manager& manager, Moses::TrellisPathList& nBestList); +//std::vector doConsensusDecoding(Moses::Manager& manager, Moses::TrellisPathList& nBestList); + +} + +#endif diff --git a/contrib/relent-filter/src/LatticeMBRGrid.cpp b/contrib/relent-filter/src/LatticeMBRGrid.cpp new file mode 100755 index 000000000..71c387839 --- /dev/null +++ b/contrib/relent-filter/src/LatticeMBRGrid.cpp @@ -0,0 +1,213 @@ +// $Id: LatticeMBRGrid.cpp 3045 2010-04-05 13:07:29Z hieuhoang1972 $ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (c) 2010 University of Edinburgh +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ +/** +* Lattice MBR grid search. Enables a grid search through the four parameters (p,r,scale and prune) used in lattice MBR. + See 'Lattice Minimum Bayes-Risk Decoding for Statistical Machine Translation by Tromble, Kumar, Och and Macherey, + EMNLP 2008 for details of the parameters. + + The grid search is controlled by specifying comma separated lists for the lmbr parameters (-lmbr-p, -lmbr-r, + -lmbr-pruning-factor and -mbr-scale). All other parameters are passed through to moses. If any of the lattice mbr + parameters are missing, then they are set to their default values. Output is of the form: + sentence-id ||| p r prune scale ||| translation-hypothesis +**/ + +#include +#include +#include +#include +#include + +#include "IOWrapper.h" +#include "LatticeMBR.h" +#include "Manager.h" +#include "StaticData.h" + + +using namespace std; +using namespace Moses; +using namespace MosesCmd; + +//keys +enum gridkey {lmbr_p,lmbr_r,lmbr_prune,lmbr_scale}; + +namespace MosesCmd +{ + +class Grid +{ +public: + /** Add a parameter with key, command line argument, and default value */ + void addParam(gridkey key, const string& arg, float defaultValue) { + m_args[arg] = key; + CHECK(m_grid.find(key) == m_grid.end()); + m_grid[key].push_back(defaultValue); + } + + /** Parse the arguments, removing those that define the grid and returning a copy of the rest */ + void parseArgs(int& argc, char**& argv) { + char** newargv = new char*[argc+1]; //Space to add mbr parameter + int newargc = 0; + for (int i = 0; i < argc; ++i) { + bool consumed = false; + for (map::const_iterator argi = m_args.begin(); argi != m_args.end(); ++argi) { + if (!strcmp(argv[i], argi->first.c_str())) { + ++i; + if (i >= argc) { + cerr << "Error: missing parameter for " << argi->first << endl; + throw runtime_error("Missing parameter"); + } else { + string value = argv[i]; + gridkey key = argi->second; + if (m_grid[key].size() != 1) { + throw runtime_error("Duplicate grid argument"); + } + m_grid[key].clear(); + char delim = ','; + string::size_type lastpos = value.find_first_not_of(delim); + string::size_type pos = value.find_first_of(delim,lastpos); + while (string::npos != pos || string::npos != lastpos) { + float param = atof(value.substr(lastpos, pos-lastpos).c_str()); + if (!param) { + cerr << "Error: Illegal grid parameter for " << argi->first << endl; + throw runtime_error("Illegal grid parameter"); + } + m_grid[key].push_back(param); + lastpos = value.find_first_not_of(delim,pos); + pos = value.find_first_of(delim,lastpos); + } + consumed = true; + } + if (consumed) break; + } + } + if (!consumed) { + newargv[newargc] = new char[strlen(argv[i]) + 1]; + strcpy(newargv[newargc],argv[i]); + ++newargc; + } + } + argc = newargc; + argv = newargv; + } + + /** Get the grid for a particular key.*/ + const vector& getGrid(gridkey key) const { + map >::const_iterator iter = m_grid.find(key); + assert (iter != m_grid.end()); + return iter->second; + + } + +private: + map > m_grid; + map m_args; +}; + +} // namespace + +int main(int argc, char* argv[]) +{ + cerr << "Lattice MBR Grid search" << endl; + + Grid grid; + grid.addParam(lmbr_p, "-lmbr-p", 0.5); + grid.addParam(lmbr_r, "-lmbr-r", 0.5); + grid.addParam(lmbr_prune, "-lmbr-pruning-factor",30.0); + grid.addParam(lmbr_scale, "-mbr-scale",1.0); + + grid.parseArgs(argc,argv); + + Parameter* params = new Parameter(); + if (!params->LoadParam(argc,argv)) { + params->Explain(); + exit(1); + } + if (!StaticData::LoadDataStatic(params, argv[0])) { + exit(1); + } + + StaticData& staticData = const_cast(StaticData::Instance()); + staticData.SetUseLatticeMBR(true); + IOWrapper* ioWrapper = GetIOWrapper(staticData); + + if (!ioWrapper) { + throw runtime_error("Failed to initialise IOWrapper"); + } + size_t nBestSize = staticData.GetMBRSize(); + + if (nBestSize <= 0) { + throw new runtime_error("Non-positive size specified for n-best list"); + } + + size_t lineCount = 0; + InputType* source = NULL; + + const vector& pgrid = grid.getGrid(lmbr_p); + const vector& rgrid = grid.getGrid(lmbr_r); + const vector& prune_grid = grid.getGrid(lmbr_prune); + const vector& scale_grid = grid.getGrid(lmbr_scale); + + while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + ++lineCount; + Sentence sentence; + const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); + Manager manager(*source,staticData.GetSearchAlgorithm(), &system); + manager.ProcessSentence(); + TrellisPathList nBestList; + manager.CalcNBest(nBestSize, nBestList,true); + //grid search + for (vector::const_iterator pi = pgrid.begin(); pi != pgrid.end(); ++pi) { + float p = *pi; + staticData.SetLatticeMBRPrecision(p); + for (vector::const_iterator ri = rgrid.begin(); ri != rgrid.end(); ++ri) { + float r = *ri; + staticData.SetLatticeMBRPRatio(r); + for (vector::const_iterator prune_i = prune_grid.begin(); prune_i != prune_grid.end(); ++prune_i) { + size_t prune = (size_t)(*prune_i); + staticData.SetLatticeMBRPruningFactor(prune); + for (vector::const_iterator scale_i = scale_grid.begin(); scale_i != scale_grid.end(); ++scale_i) { + float scale = *scale_i; + staticData.SetMBRScale(scale); + cout << lineCount << " ||| " << p << " " << r << " " << prune << " " << scale << " ||| "; + vector mbrBestHypo = doLatticeMBR(manager,nBestList); + OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(), + staticData.GetReportAllFactors(),cout); + } + } + + } + } + + + } + +} diff --git a/contrib/relent-filter/src/Main.cpp b/contrib/relent-filter/src/Main.cpp new file mode 100755 index 000000000..1f86e2cc7 --- /dev/null +++ b/contrib/relent-filter/src/Main.cpp @@ -0,0 +1,282 @@ +/*********************************************************************** +Relative Entropy-based Phrase table Pruning +Copyright (C) 2012 Wang Ling + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +/** + * Moses main, for single-threaded and multi-threaded. + **/ + +#include +#include +#include +#include + +#ifdef WIN32 +// Include Visual Leak Detector +//#include +#endif + +#include "Hypothesis.h" +#include "Manager.h" +#include "IOWrapper.h" +#include "StaticData.h" +#include "Util.h" +#include "ThreadPool.h" +#include "TranslationAnalysis.h" +#include "OutputCollector.h" +#include "RelativeEntropyCalc.h" +#include "LexicalReordering.h" +#include "LexicalReorderingState.h" + +#ifdef HAVE_PROTOBUF +#include "hypergraph.pb.h" +#endif + +using namespace std; +using namespace Moses; +using namespace MosesCmd; + +namespace MosesCmd +{ +// output floats with three significant digits +static const size_t PRECISION = 3; + +/** Enforce rounding */ +void fix(std::ostream& stream, size_t size) +{ + stream.setf(std::ios::fixed); + stream.precision(size); +} + +/** Translates a sentence. + * - calls the search (Manager) + * - applies the decision rule + * - outputs best translation and additional reporting + **/ +class TranslationTask : public Task +{ + +public: + + TranslationTask(size_t lineNumber, + InputType* source, OutputCollector* searchGraphCollector) : + m_source(source), m_lineNumber(lineNumber), + m_searchGraphCollector(searchGraphCollector) {} + + /** Translate one sentence + * gets called by main function implemented at end of this source file */ + void Run() { + + // report thread number +#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS) + TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl); +#endif + + // shorthand for "global data" + const StaticData &staticData = StaticData::Instance(); + // input sentence + Sentence sentence(); + // set translation system + const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); + + // execute the translation + // note: this executes the search, resulting in a search graph + // we still need to apply the decision rule (MAP, MBR, ...) + Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm(), &system); + manager.ProcessSentence(); + + // output search graph + if (m_searchGraphCollector) { + ostringstream out; + fix(out,PRECISION); + + vector searchGraph; + manager.GetSearchGraph(searchGraph); + out << RelativeEntropyCalc::CalcRelativeEntropy(m_lineNumber,searchGraph) << endl; + m_searchGraphCollector->Write(m_lineNumber, out.str()); + + } + manager.CalcDecoderStatistics(); + } + + ~TranslationTask() { + delete m_source; + } + +private: + InputType* m_source; + size_t m_lineNumber; + OutputCollector* m_searchGraphCollector; + std::ofstream *m_alignmentStream; + +}; + +static void PrintFeatureWeight(const FeatureFunction* ff) +{ + + size_t weightStart = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID()); + size_t weightEnd = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID()); + for (size_t i = weightStart; i < weightEnd; ++i) { + cout << ff->GetScoreProducerDescription(i-weightStart) << " " << ff->GetScoreProducerWeightShortName(i-weightStart) << " " + << StaticData::Instance().GetAllWeights()[i] << endl; + } +} + + +static void ShowWeights() +{ + fix(cout,6); + const StaticData& staticData = StaticData::Instance(); + const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); + const vector& slf =system.GetStatelessFeatureFunctions(); + const vector& sff = system.GetStatefulFeatureFunctions(); + const vector& pds = system.GetPhraseDictionaries(); + const vector& gds = system.GetGenerationDictionaries(); + for (size_t i = 0; i < sff.size(); ++i) { + PrintFeatureWeight(sff[i]); + } + for (size_t i = 0; i < slf.size(); ++i) { + PrintFeatureWeight(slf[i]); + } + for (size_t i = 0; i < pds.size(); ++i) { + PrintFeatureWeight(pds[i]); + } + for (size_t i = 0; i < gds.size(); ++i) { + PrintFeatureWeight(gds[i]); + } +} + +} //namespace + +/** main function of the command line version of the decoder **/ +int main(int argc, char** argv) +{ + try { + + // echo command line, if verbose + IFVERBOSE(1) { + TRACE_ERR("command: "); + for(int i=0; iLoadParam(argc,argv)) { + params->Explain(); + exit(1); + } + + + // initialize all "global" variables, which are stored in StaticData + // note: this also loads models such as the language model, etc. + if (!StaticData::LoadDataStatic(params, argv[0])) { + exit(1); + } + + // setting "-show-weights" -> just dump out weights and exit + if (params->isParamSpecified("show-weights")) { + ShowWeights(); + exit(0); + } + + // shorthand for accessing information in StaticData + const StaticData& staticData = StaticData::Instance(); + + + //initialise random numbers + srand(time(NULL)); + + // set up read/writing class + IOWrapper* ioWrapper = GetIOWrapper(staticData); + if (!ioWrapper) { + cerr << "Error; Failed to create IO object" << endl; + exit(1); + } + + // check on weights + vector weights = staticData.GetAllWeights(); + IFVERBOSE(2) { + TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager()); + TRACE_ERR("The global weight vector looks like this:"); + for (size_t j=0; j outputCollector; + outputCollector.reset(new OutputCollector()); + +#ifdef WITH_THREADS + ThreadPool pool(staticData.ThreadCount()); +#endif + + // main loop over set of input sentences + InputType* source = NULL; + size_t lineCount = 0; + while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + IFVERBOSE(1) { + ResetUserTime(); + } + // set up task of translating one sentence + TranslationTask* task = + new TranslationTask(lineCount,source, outputCollector.get()); + // execute task +#ifdef WITH_THREADS + pool.Submit(task); +#else + task->Run(); + delete task; +#endif + + source = NULL; //make sure it doesn't get deleted + ++lineCount; + } + + // we are done, finishing up +#ifdef WITH_THREADS + pool.Stop(true); //flush remaining jobs +#endif + + } catch (const std::exception &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return EXIT_FAILURE; + } + +#ifndef EXIT_RETURN + //This avoids that destructors are called (it can take a long time) + exit(EXIT_SUCCESS); +#else + return EXIT_SUCCESS; +#endif +} diff --git a/contrib/relent-filter/src/Main.h b/contrib/relent-filter/src/Main.h new file mode 100755 index 000000000..f0782144e --- /dev/null +++ b/contrib/relent-filter/src/Main.h @@ -0,0 +1,39 @@ +/********************************************************************* +Relative Entropy-based Phrase table Pruning +Copyright (C) 2012 Wang Ling +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#ifndef moses_cmd_Main_h +#define moses_cmd_Main_h + +#include "StaticData.h" + +class IOWrapper; + +int main(int argc, char* argv[]); +#endif diff --git a/contrib/relent-filter/src/RelativeEntropyCalc.cpp b/contrib/relent-filter/src/RelativeEntropyCalc.cpp new file mode 100755 index 000000000..212eedf87 --- /dev/null +++ b/contrib/relent-filter/src/RelativeEntropyCalc.cpp @@ -0,0 +1,83 @@ +/*********************************************************************** +Relative Entropy-based Phrase table Pruning +Copyright (C) 2012 Wang Ling + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include +#include "Hypothesis.h" +#include "StaticData.h" +#include "RelativeEntropyCalc.h" +#include "Manager.h" + +using namespace std; +using namespace Moses; +using namespace MosesCmd; + +namespace MosesCmd +{ + double RelativeEntropyCalc::CalcRelativeEntropy(int translationId, std::vector& searchGraph){ + const StaticData &staticData = StaticData::Instance(); + const Phrase *m_constraint = staticData.GetConstrainingPhrase(translationId); + + double prunedScore = -numeric_limits::max(); + double unprunedScore = -numeric_limits::max(); + for (size_t i = 0; i < searchGraph.size(); ++i) { + const SearchGraphNode& searchNode = searchGraph[i]; + int nodeId = searchNode.hypo->GetId(); + if(nodeId == 0) continue; // initial hypothesis + + int forwardId = searchNode.forward; + if(forwardId == -1){ // is final hypothesis + Phrase catOutput(0); + ConcatOutputPhraseRecursive(catOutput, searchNode.hypo); + if(catOutput == *m_constraint){ // is the output actually the same as the constraint (forced decoding does not always force the output) + const Hypothesis *prevHypo = searchNode.hypo->GetPrevHypo(); + int backId = prevHypo->GetId(); + double derivationScore = searchNode.hypo->GetScore(); + if(backId != 0){ // derivation using smaller units + if(prunedScore < derivationScore){ + prunedScore = derivationScore; + } + } + if(unprunedScore < derivationScore){ + unprunedScore = derivationScore; + } + } + } + } + + double neg_log_div = 0; + if( unprunedScore == -numeric_limits::max()){ + neg_log_div = numeric_limits::max(); // could not find phrase pair, give it a low score so that it doesnt get pruned + } + else{ + neg_log_div = unprunedScore - prunedScore; + } + if (neg_log_div > 100){ + return 100; + } + return neg_log_div; + } + + void RelativeEntropyCalc::ConcatOutputPhraseRecursive(Phrase& phrase, const Hypothesis *hypo){ + int nodeId = hypo->GetId(); + if(nodeId == 0) return; // initial hypothesis + ConcatOutputPhraseRecursive(phrase, hypo->GetPrevHypo()); + const Phrase &endPhrase = hypo->GetCurrTargetPhrase(); + phrase.Append(endPhrase); + } +} diff --git a/contrib/relent-filter/src/RelativeEntropyCalc.h b/contrib/relent-filter/src/RelativeEntropyCalc.h new file mode 100755 index 000000000..efe8ba495 --- /dev/null +++ b/contrib/relent-filter/src/RelativeEntropyCalc.h @@ -0,0 +1,51 @@ +/********************************************************************* +Relative Entropy-based Phrase table Pruning +Copyright (C) 2012 Wang Ling +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +#include +#include "Hypothesis.h" +#include "StaticData.h" +#include "Manager.h" + +using namespace std; +using namespace Moses; + +namespace MosesCmd +{ + +class RelativeEntropyCalc +{ +public: + static double CalcRelativeEntropy(int translationId, std::vector& searchGraph); + +protected: + static void ConcatOutputPhraseRecursive(Phrase& phrase, const Hypothesis *hypo); +}; + +} diff --git a/contrib/relent-filter/src/TranslationAnalysis.cpp b/contrib/relent-filter/src/TranslationAnalysis.cpp new file mode 100755 index 000000000..89da48301 --- /dev/null +++ b/contrib/relent-filter/src/TranslationAnalysis.cpp @@ -0,0 +1,126 @@ +// $Id$ + +#include +#include +#include +#include "StaticData.h" +#include "Hypothesis.h" +#include "TranslationAnalysis.h" + +using namespace Moses; + +namespace TranslationAnalysis +{ + +void PrintTranslationAnalysis(const TranslationSystem* system, std::ostream &os, const Hypothesis* hypo) +{ + os << std::endl << "TRANSLATION HYPOTHESIS DETAILS:" << std::endl; + std::vector translationPath; + + while (hypo) { + translationPath.push_back(hypo); + hypo = hypo->GetPrevHypo(); + } + + std::reverse(translationPath.begin(), translationPath.end()); + std::vector droppedWords; + std::vector::iterator tpi = translationPath.begin(); + if(tpi == translationPath.end()) + return; + ++tpi; // skip initial translation state + std::vector sourceMap; + std::vector targetMap; + std::vector lmAcc(0); + size_t lmCalls = 0; + bool doLMStats = ((*tpi)->GetLMStats() != 0); + if (doLMStats) + lmAcc.resize((*tpi)->GetLMStats()->size(), 0); + for (; tpi != translationPath.end(); ++tpi) { + std::ostringstream sms; + std::ostringstream tms; + std::string target = (*tpi)->GetTargetPhraseStringRep(); + std::string source = (*tpi)->GetSourcePhraseStringRep(); + WordsRange twr = (*tpi)->GetCurrTargetWordsRange(); + WordsRange swr = (*tpi)->GetCurrSourceWordsRange(); + const AlignmentInfo &alignmentInfo = (*tpi)->GetCurrTargetPhrase().GetAlignmentInfo(); + // language model backoff stats, + if (doLMStats) { + std::vector >& lmstats = *(*tpi)->GetLMStats(); + std::vector >::iterator i = lmstats.begin(); + std::vector::iterator acc = lmAcc.begin(); + + for (; i != lmstats.end(); ++i, ++acc) { + std::vector::iterator j = i->begin(); + lmCalls += i->size(); + for (; j != i->end(); ++j) { + (*acc) += *j; + } + } + } + + bool epsilon = false; + if (target == "") { + target=""; + epsilon = true; + droppedWords.push_back(source); + } + os << " SOURCE: " << swr << " " << source << std::endl + << " TRANSLATED AS: " << target << std::endl + << " WORD ALIGNED: " << alignmentInfo << std::endl; + size_t twr_i = twr.GetStartPos(); + size_t swr_i = swr.GetStartPos(); + if (!epsilon) { + sms << twr_i; + } + if (epsilon) { + tms << "del(" << swr_i << ")"; + } else { + tms << swr_i; + } + swr_i++; + twr_i++; + for (; twr_i <= twr.GetEndPos() && twr.GetEndPos() != NOT_FOUND; twr_i++) { + sms << '-' << twr_i; + } + for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) { + tms << '-' << swr_i; + } + if (!epsilon) targetMap.push_back(sms.str()); + sourceMap.push_back(tms.str()); + } + std::vector::iterator si = sourceMap.begin(); + std::vector::iterator ti = targetMap.begin(); + os << std::endl << "SOURCE/TARGET SPANS:"; + os << std::endl << " SOURCE:"; + for (; si != sourceMap.end(); ++si) { + os << " " << *si; + } + os << std::endl << " TARGET:"; + for (; ti != targetMap.end(); ++ti) { + os << " " << *ti; + } + os << std::endl << std::endl; + if (doLMStats && lmCalls > 0) { + std::vector::iterator acc = lmAcc.begin(); + const LMList& lmlist = system->GetLanguageModels(); + LMList::const_iterator i = lmlist.begin(); + for (; acc != lmAcc.end(); ++acc, ++i) { + char buf[256]; + sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls); + os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl; + } + } + + if (droppedWords.size() > 0) { + std::vector::iterator dwi = droppedWords.begin(); + os << std::endl << "WORDS/PHRASES DROPPED:" << std::endl; + for (; dwi != droppedWords.end(); ++dwi) { + os << "\tdropped=" << *dwi << std::endl; + } + } + os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): "; + StaticData::Instance().GetScoreIndexManager().PrintLabeledWeightedScores(os, translationPath.back()->GetScoreBreakdown(), StaticData::Instance().GetAllWeights()); + os << std::endl; +} + +} diff --git a/contrib/relent-filter/src/TranslationAnalysis.h b/contrib/relent-filter/src/TranslationAnalysis.h new file mode 100755 index 000000000..1eb7a04fd --- /dev/null +++ b/contrib/relent-filter/src/TranslationAnalysis.h @@ -0,0 +1,25 @@ +// $Id$ + +/* + * also see moses/SentenceStats + */ + +#ifndef moses_cmd_TranslationAnalysis_h +#define moses_cmd_TranslationAnalysis_h + +#include +#include "Hypothesis.h" +#include "TranslationSystem.h" + +namespace TranslationAnalysis +{ + +/*** + * print details about the translation represented in hypothesis to + * os. Included information: phrase alignment, words dropped, scores + */ +void PrintTranslationAnalysis(const Moses::TranslationSystem* system, std::ostream &os, const Moses::Hypothesis* hypo); + +} + +#endif diff --git a/contrib/relent-filter/src/mbr.cpp b/contrib/relent-filter/src/mbr.cpp new file mode 100755 index 000000000..7462d3fc6 --- /dev/null +++ b/contrib/relent-filter/src/mbr.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "TrellisPathList.h" +#include "TrellisPath.h" +#include "StaticData.h" +#include "Util.h" +#include "mbr.h" + +using namespace std ; +using namespace Moses; + + +/* Input : + 1. a sorted n-best list, with duplicates filtered out in the following format + 0 ||| amr moussa is currently on a visit to libya , tomorrow , sunday , to hold talks with regard to the in sudan . ||| 0 -4.94418 0 0 -2.16036 0 0 -81.4462 -106.593 -114.43 -105.55 -12.7873 -26.9057 -25.3715 -52.9336 7.99917 -24 ||| -4.58432 + + 2. a weight vector + 3. bleu order ( default = 4) + 4. scaling factor to weigh the weight vector (default = 1.0) + + Output : + translations that minimise the Bayes Risk of the n-best list + + +*/ + +int BLEU_ORDER = 4; +int SMOOTH = 1; +float min_interval = 1e-4; +void extract_ngrams(const vector& sentence, map < vector < const Factor* >, int > & allngrams) +{ + vector< const Factor* > ngram; + for (int k = 0; k < BLEU_ORDER; k++) { + for(int i =0; i < max((int)sentence.size()-k,0); i++) { + for ( int j = i; j<= i+k; j++) { + ngram.push_back(sentence[j]); + } + ++allngrams[ngram]; + ngram.clear(); + } + } +} + +float calculate_score(const vector< vector > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats ) +{ + int comps_n = 2*BLEU_ORDER+1; + vector comps(comps_n); + float logbleu = 0.0, brevity; + + int hyp_length = sents[hyp].size(); + + for (int i =0; i ,int > & hyp_ngrams = ngram_stats[hyp] ; + map< vector < const Factor * >, int > & ref_ngrams = ngram_stats[ref] ; + + for (map< vector< const Factor * >, int >::iterator it = hyp_ngrams.begin(); + it != hyp_ngrams.end(); it++) { + map< vector< const Factor * >, int >::iterator ref_it = ref_ngrams.find(it->first); + if(ref_it != ref_ngrams.end()) { + comps[2* (it->first.size()-1)] += min(ref_it->second,it->second); + } + } + comps[comps_n-1] = sents[ref].size(); + + for (int i=0; i 0 ) + logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH); + else + logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]); + } + logbleu /= BLEU_ORDER; + brevity = 1.0-(float)comps[comps_n-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length + if (brevity < 0.0) + logbleu += brevity; + return exp(logbleu); +} + +const TrellisPath doMBR(const TrellisPathList& nBestList) +{ + float marginal = 0; + + vector joint_prob_vec; + vector< vector > translations; + float joint_prob; + vector< map < vector , int > > ngram_stats; + + TrellisPathList::const_iterator iter; + + // get max score to prevent underflow + float maxScore = -1e20; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const TrellisPath &path = **iter; + float score = StaticData::Instance().GetMBRScale() + * path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights()); + if (maxScore < score) maxScore = score; + } + + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const TrellisPath &path = **iter; + joint_prob = UntransformScore(StaticData::Instance().GetMBRScale() * path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights()) - maxScore); + marginal += joint_prob; + joint_prob_vec.push_back(joint_prob); + + // get words in translation + vector translation; + GetOutputFactors(path, translation); + + // collect n-gram counts + map < vector < const Factor *>, int > counts; + extract_ngrams(translation,counts); + + ngram_stats.push_back(counts); + translations.push_back(translation); + } + + vector mbr_loss; + float bleu, weightedLoss; + float weightedLossCumul = 0; + float minMBRLoss = 1000000; + int minMBRLossIdx = -1; + + /* Main MBR computation done here */ + iter = nBestList.begin(); + for (unsigned int i = 0; i < nBestList.GetSize(); i++) { + weightedLossCumul = 0; + for (unsigned int j = 0; j < nBestList.GetSize(); j++) { + if ( i != j) { + bleu = calculate_score(translations, j, i,ngram_stats ); + weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal); + weightedLossCumul += weightedLoss; + if (weightedLossCumul > minMBRLoss) + break; + } + } + if (weightedLossCumul < minMBRLoss) { + minMBRLoss = weightedLossCumul; + minMBRLossIdx = i; + } + iter++; + } + /* Find sentence that minimises Bayes Risk under 1- BLEU loss */ + return nBestList.at(minMBRLossIdx); + //return translations[minMBRLossIdx]; +} + +void GetOutputFactors(const TrellisPath &path, vector &translation) +{ + const std::vector &edges = path.GetEdges(); + const std::vector& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder(); + assert (outputFactorOrder.size() == 1); + + // print the surface factor of the translation + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { + const Hypothesis &edge = *edges[currEdge]; + const Phrase &phrase = edge.GetCurrTargetPhrase(); + size_t size = phrase.GetSize(); + for (size_t pos = 0 ; pos < size ; pos++) { + + const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]); + translation.push_back(factor); + } + } +} + diff --git a/contrib/relent-filter/src/mbr.h b/contrib/relent-filter/src/mbr.h new file mode 100755 index 000000000..d08b11a98 --- /dev/null +++ b/contrib/relent-filter/src/mbr.h @@ -0,0 +1,28 @@ +// $Id$ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#ifndef moses_cmd_mbr_h +#define moses_cmd_mbr_h + +const Moses::TrellisPath doMBR(const Moses::TrellisPathList& nBestList); +void GetOutputFactors(const Moses::TrellisPath &path, std::vector &translation); +float calculate_score(const std::vector< std::vector > & sents, int ref, int hyp, std::vector < std::map < std::vector < const Moses::Factor *>, int > > & ngram_stats ); +#endif diff --git a/contrib/reranking/data/README b/contrib/reranking/data/README deleted file mode 100644 index 59b20b32d..000000000 --- a/contrib/reranking/data/README +++ /dev/null @@ -1,5 +0,0 @@ - -sample usage: - -../src/nbest -input-file nbest.small -output-file nbest.1best 1 -sort -weights weights - diff --git a/contrib/reranking/data/nbest.small b/contrib/reranking/data/nbest.small deleted file mode 100644 index 0fcbc44ce..000000000 --- a/contrib/reranking/data/nbest.small +++ /dev/null @@ -1,7 +0,0 @@ -0 ||| Once a major milestone in the Balkans ||| d: 0 -0.608213 0 0 -0.512647 0 0 lm: -35.7187 tm: -3.97053 -17.5137 -3.24082 -15.8638 2.99969 w: -7 ||| -3.92049 -0 ||| Once a crucial period in the Balkans ||| d: 0 -0.944329 0 0 -1.06468 0 0 lm: -37.5341 tm: -4.27619 -19.441 -3.81074 -14.767 3.99959 w: -7 ||| -4.00353 -1 ||| Since the world is focused on Iraq , North Korea and a possible crisis with Iran on nuclear weapons , Kosovo is somewhat unnoticed . ||| d: -6 -5.80589 -0.65383 -1.29291 -6.19413 -0.0861354 -0.993748 lm: -112.868 tm: -42.7841 -61.6487 -16.5351 -23.8061 21.9977 w: -25 ||| -13.0796 -2 ||| The public will soon turn its attention back to that province during a decision regarding his fate . ||| d: -8 -4.61691 0 -3.62979 -4.85916 0 -4.43407 lm: -81.3478 tm: -46.0407 -63.79 -23.7663 -25.175 14.9984 w: -18 ||| -12.1226 -2 ||| The public will soon be able to turn its attention back into this province during a decision on his fate . ||| d: -8 -5.53064 0 -3.51999 -3.26708 0 -4.44003 lm: -84.7939 tm: -36.2621 -66.32 -21.0804 -33.9136 13.9985 w: -21 ||| -12.1227 -2 ||| The public will soon turn his attention to them at a decision on his destiny . ||| d: -8 -5.3448 0 -2.65118 -4.35949 0 -3.95447 lm: -67.451 tm: -54.851 -89.0503 -17.9389 -22.9488 12.9986 w: -16 ||| -12.1234 -2 ||| The public will soon turn his attention to them at a decision on his destiny . ||| d: -8 -5.3448 0 -2.65118 -4.35949 0 -3.95447 lm: -67.451 tm: -54.851 -89.0503 -17.9389 -22.9488 12.9986 w: -16 ||| -12.1234 diff --git a/contrib/reranking/data/weights b/contrib/reranking/data/weights deleted file mode 100644 index c6b6c1ac0..000000000 --- a/contrib/reranking/data/weights +++ /dev/null @@ -1,11 +0,0 @@ -0 -1 2 3 -4 -5 -6 -7 -8 -9 -10 -11 -12 13 diff --git a/contrib/reranking/src/Hypo.cpp b/contrib/reranking/src/Hypo.cpp deleted file mode 100644 index 0ceb21abd..000000000 --- a/contrib/reranking/src/Hypo.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: Hypo.cpp - * basic functions to process one hypothesis - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - - -#include "Hypo.h" -#include - -//const char* NBEST_DELIM = "|||"; - -Hypo::Hypo() -{ - //cerr << "Hypo: constructor called" << endl; -} - -Hypo::~Hypo() -{ - //cerr << "Hypo: destructor called" << endl; -} - -void Hypo::Write(ofstream &outf) -{ - outf << id << NBEST_DELIM2 << trg << NBEST_DELIM2; - for (vector::iterator i = f.begin(); i != f.end(); i++) - outf << (*i) << " "; - outf << NBEST_DELIM << " " << s << endl; - -} - -float Hypo::CalcGlobal(Weights &w) -{ - //cerr << " HYP: calc global" << endl; - int sz=w.val.size(); - if (szs > h2.s); -} - diff --git a/contrib/reranking/src/Hypo.h b/contrib/reranking/src/Hypo.h deleted file mode 100644 index a85410289..000000000 --- a/contrib/reranking/src/Hypo.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: Hypo.h - * basic functions to process one hypothesis - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - - -#ifndef _HYPO_H_ -#define _HYPO_H_ - -using namespace std; - -#include -#include -#include -#include - -#include "Tools.h" - -#define NBEST_DELIM "|||" -#define NBEST_DELIM2 " ||| " - -class Hypo -{ - int id; - string trg; // translation - vector f; // feature function scores - float s; // global score - // segmentation -public: - Hypo(); - Hypo(int p_id,string &p_trg, vector &p_f, float p_s) : id(p_id),trg(p_trg),f(p_f),s(p_s) {}; - ~Hypo(); - float CalcGlobal(Weights&); - void Write(ofstream&); - bool operator< (const Hypo&) const; - // bool CompareLikelihoods (const Hypo&, const Hypo&) const; -}; - -#endif diff --git a/contrib/reranking/src/Main.cpp b/contrib/reranking/src/Main.cpp deleted file mode 100644 index 4a20b013c..000000000 --- a/contrib/reranking/src/Main.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: Main.cpp - * command line interface - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - -#include -#include -#include "ParameterNBest.h" -#include "NBest.h" -#include "Tools.h" - -#include "../../../moses/src/Util.h" - - -using namespace std; - -int main (int argc, char *argv[]) -{ - // parse parameters - ParameterNBest *parameter = new ParameterNBest(); - if (!parameter->LoadParam(argc, argv)) { - parameter->Explain(); - delete parameter; - return 1; - } - - // read input - ifstream inpf; - PARAM_VEC p=parameter->GetParam("input-file"); - if (p.size()<1 || p.size()>2) Error("The option -input-file requires one or two arguments"); - int in_n=p.size()>1 ? Moses::Scan(p[1]) : 0; - cout << "NBest version 0.1, written by Holger.Schwenk@lium.univ-lemans.fr" << endl - << " - reading input from file '" << p[0] << "'"; - if (in_n>0) cout << " (limited to the first " << in_n << " hypothesis)"; - cout << endl; - inpf.open(p[0].c_str()); - if (inpf.fail()) { - perror ("ERROR"); - exit(1); - } - - // open output - ofstream outf; - p=parameter->GetParam("output-file"); - if (p.size()<1 || p.size()>2) Error("The option -output-file requires one or two arguments"); - int out_n=p.size()>1 ? Moses::Scan(p[1]) : 0; - cout << " - writing output to file '" << p[0] << "'"; - if (out_n>0) cout << " (limited to the first " << out_n << " hypothesis)"; - cout << endl; - outf.open(p[0].c_str()); - if (outf.fail()) { - perror ("ERROR"); - exit(1); - } - - // eventually read weights - Weights w; - int do_calc=false; - if (parameter->isParamSpecified("weights")) { - p=parameter->GetParam("weights"); - if (p.size()<1) Error("The option -weights requires one argument"); - cout << " - reading weights from file '" << p[0] << "'"; - int n=w.Read(p[0].c_str()); - cout << " (found " << n << " values)" << endl; - do_calc=true; - cout << " - recalculating global scores" << endl; - } - - // shall we sort ? - bool do_sort = parameter->isParamSpecified("sort"); - if (do_sort) cout << " - sorting global scores" << endl; - - // main loop - int nb_sent=0, nb_nbest=0; - while (!inpf.eof()) { - NBest nbest(inpf, in_n); - - if (do_calc) nbest.CalcGlobal(w); - if (do_sort) nbest.Sort(); - nbest.Write(outf, out_n); - - nb_sent++; - nb_nbest+=nbest.NbNBest(); - } - inpf.close(); - outf.close(); - - // display final statistics - cout << " - processed " << nb_nbest << " n-best hypotheses in " << nb_sent << " sentences" - << " (average " << (float) nb_nbest/nb_sent << ")" << endl; - - return 0; -} diff --git a/contrib/reranking/src/Makefile b/contrib/reranking/src/Makefile deleted file mode 100644 index c2711741e..000000000 --- a/contrib/reranking/src/Makefile +++ /dev/null @@ -1,18 +0,0 @@ - -# where to find include files and libraries from Moses -MOSES_INC=../../../moses/src ../../.. -LIB_DIR=../../../moses/src/ - -LIBS=-lmoses -lz -OBJS=Main.o NBest.o Hypo.o Tools.o ParameterNBest.o - -CFLAGS=-I$(MOSES_INC) - -nbest-tool: $(OBJS) - $(CXX) -o nbest $(OBJS) -L$(LIB_DIR) $(LIBS) - -%.o: %.cpp - $(CXX) $(CFLAGS) -o $@ -c $< - -clean: - -rm $(OBJS) nbest diff --git a/contrib/reranking/src/NBest.cpp b/contrib/reranking/src/NBest.cpp deleted file mode 100644 index 24a0f60c3..000000000 --- a/contrib/reranking/src/NBest.cpp +++ /dev/null @@ -1,131 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: NBest.cpp - * basic functions on n-best lists - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - - -#include "NBest.h" - -#include "Util.h" // from Moses - -#include -#include - -//NBest::NBest() { -//cerr << "NBEST: constructor called" << endl; -//} - - -bool NBest::ParseLine(ifstream &inpf, const int n) -{ - static string line; // used internally to buffer an input line - static int prev_id=-1; // used to detect a change of the n-best ID - int id; - vector f; - float s; - int pos=0, epos; - vector blocks; - - - if (line.empty()) { - getline(inpf,line); - if (inpf.eof()) return false; - } - - // split line into blocks - //cerr << "PARSE line: " << line << endl; - while ((epos=line.find(NBEST_DELIM,pos))!=string::npos) { - blocks.push_back(line.substr(pos,epos-pos)); - // cerr << " block: " << blocks.back() << endl; - pos=epos+strlen(NBEST_DELIM); - } - blocks.push_back(line.substr(pos,line.size())); - // cerr << " block: " << blocks.back() << endl; - - if (blocks.size()<4) { - cerr << line << endl; - Error("can't parse the above line"); - } - - // parse ID - id=Scan(blocks[0]); - if (prev_id>=0 && id!=prev_id) { - prev_id=id; // new nbest list has started - return false; - } - prev_id=id; - //cerr << "same ID " << id << endl; - - if (n>0 && nbest.size() >= n) { - //cerr << "skipped" << endl; - line.clear(); - return true; // skip parsing of unused hypos - } - - // parse feature function scores - //cerr << "PARSE features: '" << blocks[2] << "' size: " << blocks[2].size() << endl; - pos=blocks[2].find_first_not_of(' '); - while (pos(feat)); - //cerr << " value: " << f.back() << endl; - } - pos=epos+1; - } - - // eventually parse segmentation - if (blocks.size()>4) { - Error("parsing segmentation not yet supported"); - } - - nbest.push_back(Hypo(id, blocks[1], f, Scan(blocks[3]))); - - line.clear(); // force read of new line - - return true; -} - - -NBest::NBest(ifstream &inpf, const int n) -{ - //cerr << "NBEST: constructor with file called" << endl; - while (ParseLine(inpf,n)); - //cerr << "NBEST: found " << nbest.size() << " lines" << endl; -} - - -NBest::~NBest() -{ - //cerr << "NBEST: destructor called" << endl; -} - -void NBest::Write(ofstream &outf, int n) -{ - if (n<1 || n>nbest.size()) n=nbest.size(); - for (int i=0; i::iterator i = nbest.begin(); i != nbest.end(); i++) { - (*i).CalcGlobal(w); - } -} - - -void NBest::Sort() -{ - sort(nbest.begin(),nbest.end()); -} - diff --git a/contrib/reranking/src/NBest.h b/contrib/reranking/src/NBest.h deleted file mode 100644 index 9a4aa9447..000000000 --- a/contrib/reranking/src/NBest.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: NBest.h - * basic functions on n-best lists - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - - -#ifndef _NBEST_H_ -#define _NBEST_H_ - -using namespace std; - -#include -#include -#include -#include - -#include "Tools.h" -#include "Hypo.h" - -class NBest -{ - int id; - string src; - vector nbest; - bool ParseLine(ifstream &inpf, const int n); -public: - NBest(ifstream&, const int=0); - ~NBest(); - int NbNBest() { - return nbest.size(); - }; - float CalcGlobal(Weights&); - void Sort(); // largest values first - void Write(ofstream&, int=0); -}; - -void Error(char *msg); - -#endif diff --git a/contrib/reranking/src/ParameterNBest.cpp b/contrib/reranking/src/ParameterNBest.cpp deleted file mode 100644 index 005f3890c..000000000 --- a/contrib/reranking/src/ParameterNBest.cpp +++ /dev/null @@ -1,337 +0,0 @@ -// $Id: $ - -/*********************************************************************** -nbest - tool to process Moses n-best list -Copyright (C) 2008 Holger Schwenk, University of Le Mans, France - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -***********************************************************************/ - -#include -#include -#include -#include -#include -#include "ParameterNBest.h" -#include "Tools.h" - -#include "Util.h" // from Moses -#include "InputFileStream.h" -#include "UserMessage.h" - -using namespace std; - -/** define allowed parameters */ -ParameterNBest::ParameterNBest() -{ - AddParam("input-file", "i", "file name of the input n-best list"); - AddParam("output-file", "o", "file name of the output n-best list"); - AddParam("recalc", "r", "recalc global scores"); - AddParam("weights", "w", "coefficients of the feature functions"); - AddParam("sort", "s", "sort n-best list according to the global scores"); - AddParam("lexical", "l", "report number of lexically different hypothesis"); -} - -ParameterNBest::~ParameterNBest() -{ -} - -/** initialize a parameter, sub of constructor */ -void ParameterNBest::AddParam(const string ¶mName, const string &description) -{ - m_valid[paramName] = true; - m_description[paramName] = description; -} - -/** initialize a parameter (including abbreviation), sub of constructor */ -void ParameterNBest::AddParam(const string ¶mName, const string &abbrevName, const string &description) -{ - m_valid[paramName] = true; - m_valid[abbrevName] = true; - m_abbreviation[paramName] = abbrevName; - m_description[paramName] = description; -} - -/** print descriptions of all parameters */ -void ParameterNBest::Explain() -{ - cerr << "Usage:" << endl; - for(PARAM_STRING::const_iterator iterParam = m_description.begin(); iterParam != m_description.end(); iterParam++) { - const string paramName = iterParam->first; - const string paramDescription = iterParam->second; - cerr << "\t-" << paramName; - PARAM_STRING::const_iterator iterAbbr = m_abbreviation.find( paramName ); - if ( iterAbbr != m_abbreviation.end() ) - cerr << " (" << iterAbbr->second << ")"; - cerr << ": " << paramDescription << endl; - } -} - -/** check whether an item on the command line is a switch or a value - * \param token token on the command line to checked **/ - -bool ParameterNBest::isOption(const char* token) -{ - if (! token) return false; - std::string tokenString(token); - size_t length = tokenString.size(); - if (length > 0 && tokenString.substr(0,1) != "-") return false; - if (length > 1 && tokenString.substr(1,1).find_first_not_of("0123456789") == 0) return true; - return false; -} - -/** load all parameters from the configuration file and the command line switches */ -bool ParameterNBest::LoadParam(const string &filePath) -{ - const char *argv[] = {"executable", "-f", filePath.c_str() }; - return LoadParam(3, (char**) argv); -} - -/** load all parameters from the configuration file and the command line switches */ -bool ParameterNBest::LoadParam(int argc, char* argv[]) -{ - // config file (-f) arg mandatory - string configPath; - /* - if ( (configPath = FindParam("-f", argc, argv)) == "" - && (configPath = FindParam("-config", argc, argv)) == "") - { - PrintCredit(); - - UserMessage::Add("No configuration file was specified. Use -config or -f"); - return false; - } - else - { - if (!ReadConfigFile(configPath)) - { - UserMessage::Add("Could not read "+configPath); - return false; - } - } - */ - - // overwrite parameters with values from switches - for(PARAM_STRING::const_iterator iterParam = m_description.begin(); iterParam != m_description.end(); iterParam++) { - const string paramName = iterParam->first; - OverwriteParam("-" + paramName, paramName, argc, argv); - } - - // ... also shortcuts - for(PARAM_STRING::const_iterator iterParam = m_abbreviation.begin(); iterParam != m_abbreviation.end(); iterParam++) { - const string paramName = iterParam->first; - const string paramShortName = iterParam->second; - OverwriteParam("-" + paramShortName, paramName, argc, argv); - } - - // logging of parameters that were set in either config or switch - int verbose = 1; - if (m_setting.find("verbose") != m_setting.end() && - m_setting["verbose"].size() > 0) - verbose = Scan(m_setting["verbose"][0]); - if (verbose >= 1) { // only if verbose - TRACE_ERR( "Defined parameters (per moses.ini or switch):" << endl); - for(PARAM_MAP::const_iterator iterParam = m_setting.begin() ; iterParam != m_setting.end(); iterParam++) { - TRACE_ERR( "\t" << iterParam->first << ": "); - for ( size_t i = 0; i < iterParam->second.size(); i++ ) - TRACE_ERR( iterParam->second[i] << " "); - TRACE_ERR( endl); - } - } - - // check for illegal parameters - bool noErrorFlag = true; - for (int i = 0 ; i < argc ; i++) { - if (isOption(argv[i])) { - string paramSwitch = (string) argv[i]; - string paramName = paramSwitch.substr(1); - if (m_valid.find(paramName) == m_valid.end()) { - UserMessage::Add("illegal switch: " + paramSwitch); - noErrorFlag = false; - } - } - } - - // check if parameters make sense - return Validate() && noErrorFlag; -} - -/** check that parameter settings make sense */ -bool ParameterNBest::Validate() -{ - bool noErrorFlag = true; - - // required parameters - if (m_setting["input-file"].size() == 0) { - UserMessage::Add("No input-file"); - noErrorFlag = false; - } - - if (m_setting["output-file"].size() == 0) { - UserMessage::Add("No output-file"); - noErrorFlag = false; - } - - if (m_setting["recalc"].size() > 0 && m_setting["weights"].size()==0) { - UserMessage::Add("you need to spezify weight when recalculating global scores"); - noErrorFlag = false; - } - - - return noErrorFlag; -} - -/** check whether a file exists */ -bool ParameterNBest::FilesExist(const string ¶mName, size_t tokenizeIndex,std::vector const& extensions) -{ - typedef std::vector StringVec; - StringVec::const_iterator iter; - - PARAM_MAP::const_iterator iterParam = m_setting.find(paramName); - if (iterParam == m_setting.end()) { - // no param. therefore nothing to check - return true; - } - const StringVec &pathVec = (*iterParam).second; - for (iter = pathVec.begin() ; iter != pathVec.end() ; ++iter) { - StringVec vec = Tokenize(*iter); - if (tokenizeIndex >= vec.size()) { - stringstream errorMsg(""); - errorMsg << "Expected at least " << (tokenizeIndex+1) << " tokens per emtry in '" - << paramName << "', but only found " - << vec.size(); - UserMessage::Add(errorMsg.str()); - return false; - } - const string &pathStr = vec[tokenizeIndex]; - - bool fileFound=0; - for(size_t i=0; i (size_t)index) - m_setting[paramName][index] = argv[startPos]; - else - m_setting[paramName].push_back(argv[startPos]); - index++; - startPos++; - } -} - - -/** read parameters from a configuration file */ -bool ParameterNBest::ReadConfigFile( string filePath ) -{ - InputFileStream inFile(filePath); - string line, paramName; - while(getline(inFile, line)) { - // comments - size_t comPos = line.find_first_of("#"); - if (comPos != string::npos) - line = line.substr(0, comPos); - // trim leading and trailing spaces/tabs - line = Trim(line); - - if (line[0]=='[') { - // new parameter - for (size_t currPos = 0 ; currPos < line.size() ; currPos++) { - if (line[currPos] == ']') { - paramName = line.substr(1, currPos - 1); - break; - } - } - } else if (line != "") { - // add value to parameter - m_setting[paramName].push_back(line); - } - } - return true; -} - - -void ParameterNBest::PrintCredit() -{ - cerr << "NBest - A tool to process Moses n-best lists" << endl - << "Copyright (C) 2008 Holger Schwenk" << endl << endl - - << "This library is free software; you can redistribute it and/or" << endl - << "modify it under the terms of the GNU Lesser General Public" << endl - << "License as published by the Free Software Foundation; either" << endl - << "version 2.1 of the License, or (at your option) any later version." << endl << endl - - << "This library is distributed in the hope that it will be useful," << endl - << "but WITHOUT ANY WARRANTY; without even the implied warranty of" << endl - << "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU" << endl - << "Lesser General Public License for more details." << endl << endl - - << "You should have received a copy of the GNU Lesser General Public" << endl - << "License along with this library; if not, write to the Free Software" << endl - << "Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA" << endl << endl - << "***********************************************************************" << endl << endl - << "Built on " << __DATE__ << endl << endl - - << "Written by Holger Schwenk, Holger.Schwenk@lium.univ-lemans.fr" << endl << endl; -} - diff --git a/contrib/reranking/src/ParameterNBest.h b/contrib/reranking/src/ParameterNBest.h deleted file mode 100644 index bc554d4b9..000000000 --- a/contrib/reranking/src/ParameterNBest.h +++ /dev/null @@ -1,76 +0,0 @@ -// $Id: $ - -/*********************************************************************** -nbest - tool to process Moses n-best list -Copyright (C) 2008 Holger Schwenk, University of Le Mans, France - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -***********************************************************************/ - -#ifndef _PARAMETER_NBEST_H_ -#define _PARAMETER_NBEST_H_ - -#include -#include -#include -#include "TypeDef.h" - -typedef std::vector PARAM_VEC; -typedef std::map PARAM_MAP; -typedef std::map PARAM_BOOL; -typedef std::map PARAM_STRING; - -/** Handles parameter values set in config file or on command line. - * Process raw parameter data (names and values as strings) for StaticData - * to parse; to get useful values, see StaticData. */ -class ParameterNBest -{ -protected: - PARAM_MAP m_setting; - PARAM_BOOL m_valid; - PARAM_STRING m_abbreviation; - PARAM_STRING m_description; - - std::string FindParam(const std::string ¶mSwitch, int argc, char* argv[]); - void OverwriteParam(const std::string ¶mSwitch, const std::string ¶mName, int argc, char* argv[]); - bool ReadConfigFile( std::string filePath ); - bool FilesExist(const std::string ¶mName, size_t tokenizeIndex,std::vector const& fileExtension=std::vector(1,"")); - bool isOption(const char* token); - bool Validate(); - - void AddParam(const std::string ¶mName, const std::string &description); - void AddParam(const std::string ¶mName, const std::string &abbrevName, const std::string &description); - - void PrintCredit(); - -public: - ParameterNBest(); - ~ParameterNBest(); - bool LoadParam(int argc, char* argv[]); - bool LoadParam(const std::string &filePath); - void Explain(); - - /** return a vector of strings holding the whitespace-delimited values on the ini-file line corresponding to the given parameter name */ - const PARAM_VEC &GetParam(const std::string ¶mName) { - return m_setting[paramName]; - } - /** check if parameter is defined (either in moses.ini or as switch) */ - bool isParamSpecified(const std::string ¶mName) { - return m_setting.find( paramName ) != m_setting.end(); - } - -}; - -#endif diff --git a/contrib/reranking/src/Tools.cpp b/contrib/reranking/src/Tools.cpp deleted file mode 100644 index 8312c3370..000000000 --- a/contrib/reranking/src/Tools.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: Tools.cpp - * basic utility functions - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - -#include "Tools.h" - -int Weights::Read(const char *fname) -{ - ifstream inpf; - - inpf.open(fname); - if (inpf.fail()) { - perror ("ERROR"); - exit(1); - } - - float f; - while (inpf >> f) val.push_back(f); - - inpf.close(); - return val.size(); -} - diff --git a/contrib/reranking/src/Tools.h b/contrib/reranking/src/Tools.h deleted file mode 100644 index eb71746b0..000000000 --- a/contrib/reranking/src/Tools.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * nbest: tool to process moses n-best lists - * - * File: Tools.cpp - * basic utility functions - * - * Created by Holger Schwenk, University of Le Mans, 05/16/2008 - * - */ - - -#ifndef _TOOLS_H_ -#define _TOOLS_H_ - -using namespace std; - -#include -#include -#include - -class Weights -{ - vector val; -public: - Weights() {}; - ~Weights() {}; - int Read(const char *); - friend class Hypo; -}; - -//****************************************************** - -/* -template -inline T Scan(const std::string &input) -{ - std::stringstream stream(input); - T ret; - stream >> ret; - return ret; -} -*/ - -//****************************************************** - -inline void Error (char *msg) -{ - cerr << "ERROR: " << msg << endl; - exit(1); -} - -//****************************************************** -// From Moses code: - - -/* - * Outputting debugging/verbose information to stderr. - * Use TRACE_ENABLE flag to redirect tracing output into oblivion - * so that you can output your own ad-hoc debugging info. - * However, if you use stderr diretly, please delete calls to it once - * you finished debugging so that it won't clutter up. - * Also use TRACE_ENABLE to turn off output of any debugging info - * when compiling for a gui front-end so that running gui won't generate - * output on command line - * */ -#ifdef TRACE_ENABLE -#define TRACE_ERR(str) std::cerr << str -#else -#define TRACE_ERR(str) {} -#endif - -#endif - diff --git a/contrib/server/Translation-web/src/conf/MANIFEST.MF b/contrib/server/Translation-web/src/conf/MANIFEST.MF new file mode 100755 index 000000000..58630c02e --- /dev/null +++ b/contrib/server/Translation-web/src/conf/MANIFEST.MF @@ -0,0 +1,2 @@ +Manifest-Version: 1.0 + diff --git a/contrib/server/Translation-web/src/java/com/hpl/mt/Translate.java b/contrib/server/Translation-web/src/java/com/hpl/mt/Translate.java new file mode 100755 index 000000000..b0823431d --- /dev/null +++ b/contrib/server/Translation-web/src/java/com/hpl/mt/Translate.java @@ -0,0 +1,129 @@ +package com.hpl.mt; + +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URL; +import java.util.HashMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.xmlrpc.XmlRpcException; +import org.apache.xmlrpc.client.XmlRpcClient; +import org.apache.xmlrpc.client.XmlRpcClientConfigImpl; + +/** + * + * @author ulanov + */ +public class Translate extends HttpServlet { + + /** + * Processes requests for both HTTP + * GET and + * POST methods. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + protected void processRequest(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.setContentType("text/html;charset=UTF-8"); + System.out.println("before" + request.getCharacterEncoding()); + request.setCharacterEncoding("UTF-8"); + System.out.println("after" + request.getCharacterEncoding()); + PrintWriter out = response.getWriter(); + try { + /* + * TODO output your page here. You may use following sample code. + */ + // Create an instance of XmlRpcClient + String textToTranslate = request.getParameter("text"); + XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl(); + config.setServerURL(new URL("http://localhost:9008/RPC2")); + XmlRpcClient client = new XmlRpcClient(); + client.setConfig(config); + // The XML-RPC data type used by mosesserver is . In Java, this data type can be represented using HashMap. + HashMap mosesParams = new HashMap(); + mosesParams.put("text", textToTranslate); + mosesParams.put("align", "true"); + mosesParams.put("report-all-factors", "true"); + // The XmlRpcClient.execute method doesn't accept Hashmap (pParams). It's either Object[] or List. + Object[] params = new Object[] { null }; + params[0] = mosesParams; + // Invoke the remote method "translate". The result is an Object, convert it to a HashMap. + HashMap result; + try { + result = (HashMap)client.execute("translate", params); + } catch (XmlRpcException ex) { + Logger.getLogger(Translate.class.getName()).log(Level.SEVERE, null, ex); + throw new IOException("XML-RPC failed"); + } + // Print the returned results + String textTranslation = (String)result.get("text"); + System.out.println("Input : "+textToTranslate); + System.out.println("Translation : "+textTranslation); + out.write(textTranslation); + if (result.get("align") != null){ + Object[] aligns = (Object[])result.get("align"); + System.out.println("Phrase alignments : [Source Start:Source End][Target Start]"); + for ( Object element : aligns) { + HashMap align = (HashMap)element; + System.out.println("["+align.get("src-start")+":"+align.get("src-end")+"]["+align.get("tgt-start")+"]"); + } + } + } finally { + out.close(); + } + } + + // + /** + * Handles the HTTP + * GET method. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + processRequest(request, response); + } + + /** + * Handles the HTTP + * POST method. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + processRequest(request, response); + } + + /** + * Returns a short description of the servlet. + * + * @return a String containing servlet description + */ + @Override + public String getServletInfo() { + return "Short description"; + }// +} diff --git a/contrib/server/Translation-web/web/META-INF/context.xml b/contrib/server/Translation-web/web/META-INF/context.xml new file mode 100755 index 000000000..9772ce4a1 --- /dev/null +++ b/contrib/server/Translation-web/web/META-INF/context.xml @@ -0,0 +1,2 @@ + + diff --git a/contrib/server/Translation-web/web/WEB-INF/web.xml b/contrib/server/Translation-web/web/WEB-INF/web.xml new file mode 100755 index 000000000..4147aafae --- /dev/null +++ b/contrib/server/Translation-web/web/WEB-INF/web.xml @@ -0,0 +1,16 @@ + + + + Translate + com.hpl.mt.Translate + + + Translate + /Translate + + + + 30 + + + diff --git a/contrib/server/Translation-web/web/css/common.css b/contrib/server/Translation-web/web/css/common.css new file mode 100755 index 000000000..c379ac161 --- /dev/null +++ b/contrib/server/Translation-web/web/css/common.css @@ -0,0 +1,22 @@ +/* + Document : common + Created on : Jul 31, 2012, 11:53:29 AM + Author : ulanov + Description: + Purpose of the stylesheet follows. +*/ + +root { + display: block; +} + +body {font-size:small; font-family: Verdana,Arial,sans-serif;height:auto; width: auto;} +span {font-size:medium;} + +#north_tab {height: 10%; width: 100%; float: top;} +#south_tab {height: 80%; width: 100%; float: bottom;} + +#input_text {height: 50%; width: 30%; margin-right: 10px; float: left;} +#output_text {height: 50%; width: 30%; margin-right: 10px; float: left;} + +#translate {float: left; margin-right: 10px;} diff --git a/contrib/server/Translation-web/web/index.html b/contrib/server/Translation-web/web/index.html new file mode 100755 index 000000000..dd7934739 --- /dev/null +++ b/contrib/server/Translation-web/web/index.html @@ -0,0 +1,47 @@ + + + + + + + + + +Translate FR-EN + + + +
+

Translate FR-EN

+
+
+ + + + + +
+ + + \ No newline at end of file diff --git a/contrib/server/Translation-web/web/lib/jquery-1.6.4.js b/contrib/server/Translation-web/web/lib/jquery-1.6.4.js new file mode 100755 index 000000000..7a160217c --- /dev/null +++ b/contrib/server/Translation-web/web/lib/jquery-1.6.4.js @@ -0,0 +1,9046 @@ +/*! + * jQuery JavaScript Library v1.6.4 + * http://jquery.com/ + * + * Copyright 2011, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2011, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Mon Sep 12 18:54:48 2011 -0400 + */ +(function( window, undefined ) { + +// Use the correct document accordingly with window argument (sandbox) +var document = window.document, + navigator = window.navigator, + location = window.location; +var jQuery = (function() { + +// Define a local copy of jQuery +var jQuery = function( selector, context ) { + // The jQuery object is actually just the init constructor 'enhanced' + return new jQuery.fn.init( selector, context, rootjQuery ); + }, + + // Map over jQuery in case of overwrite + _jQuery = window.jQuery, + + // Map over the $ in case of overwrite + _$ = window.$, + + // A central reference to the root jQuery(document) + rootjQuery, + + // A simple way to check for HTML strings or ID strings + // Prioritize #id over to avoid XSS via location.hash (#9521) + quickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/, + + // Check if a string has a non-whitespace character in it + rnotwhite = /\S/, + + // Used for trimming whitespace + trimLeft = /^\s+/, + trimRight = /\s+$/, + + // Check for digits + rdigit = /\d/, + + // Match a standalone tag + rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>)?$/, + + // JSON RegExp + rvalidchars = /^[\],:{}\s]*$/, + rvalidescape = /\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g, + rvalidtokens = /"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g, + rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g, + + // Useragent RegExp + rwebkit = /(webkit)[ \/]([\w.]+)/, + ropera = /(opera)(?:.*version)?[ \/]([\w.]+)/, + rmsie = /(msie) ([\w.]+)/, + rmozilla = /(mozilla)(?:.*? rv:([\w.]+))?/, + + // Matches dashed string for camelizing + rdashAlpha = /-([a-z]|[0-9])/ig, + rmsPrefix = /^-ms-/, + + // Used by jQuery.camelCase as callback to replace() + fcamelCase = function( all, letter ) { + return ( letter + "" ).toUpperCase(); + }, + + // Keep a UserAgent string for use with jQuery.browser + userAgent = navigator.userAgent, + + // For matching the engine and version of the browser + browserMatch, + + // The deferred used on DOM ready + readyList, + + // The ready event handler + DOMContentLoaded, + + // Save a reference to some core methods + toString = Object.prototype.toString, + hasOwn = Object.prototype.hasOwnProperty, + push = Array.prototype.push, + slice = Array.prototype.slice, + trim = String.prototype.trim, + indexOf = Array.prototype.indexOf, + + // [[Class]] -> type pairs + class2type = {}; + +jQuery.fn = jQuery.prototype = { + constructor: jQuery, + init: function( selector, context, rootjQuery ) { + var match, elem, ret, doc; + + // Handle $(""), $(null), or $(undefined) + if ( !selector ) { + return this; + } + + // Handle $(DOMElement) + if ( selector.nodeType ) { + this.context = this[0] = selector; + this.length = 1; + return this; + } + + // The body element only exists once, optimize finding it + if ( selector === "body" && !context && document.body ) { + this.context = document; + this[0] = document.body; + this.selector = selector; + this.length = 1; + return this; + } + + // Handle HTML strings + if ( typeof selector === "string" ) { + // Are we dealing with HTML string or an ID? + if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { + // Assume that strings that start and end with <> are HTML and skip the regex check + match = [ null, selector, null ]; + + } else { + match = quickExpr.exec( selector ); + } + + // Verify a match, and that no context was specified for #id + if ( match && (match[1] || !context) ) { + + // HANDLE: $(html) -> $(array) + if ( match[1] ) { + context = context instanceof jQuery ? context[0] : context; + doc = (context ? context.ownerDocument || context : document); + + // If a single string is passed in and it's a single tag + // just do a createElement and skip the rest + ret = rsingleTag.exec( selector ); + + if ( ret ) { + if ( jQuery.isPlainObject( context ) ) { + selector = [ document.createElement( ret[1] ) ]; + jQuery.fn.attr.call( selector, context, true ); + + } else { + selector = [ doc.createElement( ret[1] ) ]; + } + + } else { + ret = jQuery.buildFragment( [ match[1] ], [ doc ] ); + selector = (ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment).childNodes; + } + + return jQuery.merge( this, selector ); + + // HANDLE: $("#id") + } else { + elem = document.getElementById( match[2] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id !== match[2] ) { + return rootjQuery.find( selector ); + } + + // Otherwise, we inject the element directly into the jQuery object + this.length = 1; + this[0] = elem; + } + + this.context = document; + this.selector = selector; + return this; + } + + // HANDLE: $(expr, $(...)) + } else if ( !context || context.jquery ) { + return (context || rootjQuery).find( selector ); + + // HANDLE: $(expr, context) + // (which is just equivalent to: $(context).find(expr) + } else { + return this.constructor( context ).find( selector ); + } + + // HANDLE: $(function) + // Shortcut for document ready + } else if ( jQuery.isFunction( selector ) ) { + return rootjQuery.ready( selector ); + } + + if (selector.selector !== undefined) { + this.selector = selector.selector; + this.context = selector.context; + } + + return jQuery.makeArray( selector, this ); + }, + + // Start with an empty selector + selector: "", + + // The current version of jQuery being used + jquery: "1.6.4", + + // The default length of a jQuery object is 0 + length: 0, + + // The number of elements contained in the matched element set + size: function() { + return this.length; + }, + + toArray: function() { + return slice.call( this, 0 ); + }, + + // Get the Nth element in the matched element set OR + // Get the whole matched element set as a clean array + get: function( num ) { + return num == null ? + + // Return a 'clean' array + this.toArray() : + + // Return just the object + ( num < 0 ? this[ this.length + num ] : this[ num ] ); + }, + + // Take an array of elements and push it onto the stack + // (returning the new matched element set) + pushStack: function( elems, name, selector ) { + // Build a new jQuery matched element set + var ret = this.constructor(); + + if ( jQuery.isArray( elems ) ) { + push.apply( ret, elems ); + + } else { + jQuery.merge( ret, elems ); + } + + // Add the old object onto the stack (as a reference) + ret.prevObject = this; + + ret.context = this.context; + + if ( name === "find" ) { + ret.selector = this.selector + (this.selector ? " " : "") + selector; + } else if ( name ) { + ret.selector = this.selector + "." + name + "(" + selector + ")"; + } + + // Return the newly-formed element set + return ret; + }, + + // Execute a callback for every element in the matched set. + // (You can seed the arguments with an array of args, but this is + // only used internally.) + each: function( callback, args ) { + return jQuery.each( this, callback, args ); + }, + + ready: function( fn ) { + // Attach the listeners + jQuery.bindReady(); + + // Add the callback + readyList.done( fn ); + + return this; + }, + + eq: function( i ) { + return i === -1 ? + this.slice( i ) : + this.slice( i, +i + 1 ); + }, + + first: function() { + return this.eq( 0 ); + }, + + last: function() { + return this.eq( -1 ); + }, + + slice: function() { + return this.pushStack( slice.apply( this, arguments ), + "slice", slice.call(arguments).join(",") ); + }, + + map: function( callback ) { + return this.pushStack( jQuery.map(this, function( elem, i ) { + return callback.call( elem, i, elem ); + })); + }, + + end: function() { + return this.prevObject || this.constructor(null); + }, + + // For internal use only. + // Behaves like an Array's method, not like a jQuery method. + push: push, + sort: [].sort, + splice: [].splice +}; + +// Give the init function the jQuery prototype for later instantiation +jQuery.fn.init.prototype = jQuery.fn; + +jQuery.extend = jQuery.fn.extend = function() { + var options, name, src, copy, copyIsArray, clone, + target = arguments[0] || {}, + i = 1, + length = arguments.length, + deep = false; + + // Handle a deep copy situation + if ( typeof target === "boolean" ) { + deep = target; + target = arguments[1] || {}; + // skip the boolean and the target + i = 2; + } + + // Handle case when target is a string or something (possible in deep copy) + if ( typeof target !== "object" && !jQuery.isFunction(target) ) { + target = {}; + } + + // extend jQuery itself if only one argument is passed + if ( length === i ) { + target = this; + --i; + } + + for ( ; i < length; i++ ) { + // Only deal with non-null/undefined values + if ( (options = arguments[ i ]) != null ) { + // Extend the base object + for ( name in options ) { + src = target[ name ]; + copy = options[ name ]; + + // Prevent never-ending loop + if ( target === copy ) { + continue; + } + + // Recurse if we're merging plain objects or arrays + if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { + if ( copyIsArray ) { + copyIsArray = false; + clone = src && jQuery.isArray(src) ? src : []; + + } else { + clone = src && jQuery.isPlainObject(src) ? src : {}; + } + + // Never move original objects, clone them + target[ name ] = jQuery.extend( deep, clone, copy ); + + // Don't bring in undefined values + } else if ( copy !== undefined ) { + target[ name ] = copy; + } + } + } + } + + // Return the modified object + return target; +}; + +jQuery.extend({ + noConflict: function( deep ) { + if ( window.$ === jQuery ) { + window.$ = _$; + } + + if ( deep && window.jQuery === jQuery ) { + window.jQuery = _jQuery; + } + + return jQuery; + }, + + // Is the DOM ready to be used? Set to true once it occurs. + isReady: false, + + // A counter to track how many items to wait for before + // the ready event fires. See #6781 + readyWait: 1, + + // Hold (or release) the ready event + holdReady: function( hold ) { + if ( hold ) { + jQuery.readyWait++; + } else { + jQuery.ready( true ); + } + }, + + // Handle when the DOM is ready + ready: function( wait ) { + // Either a released hold or an DOMready/load event and not yet ready + if ( (wait === true && !--jQuery.readyWait) || (wait !== true && !jQuery.isReady) ) { + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( !document.body ) { + return setTimeout( jQuery.ready, 1 ); + } + + // Remember that the DOM is ready + jQuery.isReady = true; + + // If a normal DOM Ready event fired, decrement, and wait if need be + if ( wait !== true && --jQuery.readyWait > 0 ) { + return; + } + + // If there are functions bound, to execute + readyList.resolveWith( document, [ jQuery ] ); + + // Trigger any bound ready events + if ( jQuery.fn.trigger ) { + jQuery( document ).trigger( "ready" ).unbind( "ready" ); + } + } + }, + + bindReady: function() { + if ( readyList ) { + return; + } + + readyList = jQuery._Deferred(); + + // Catch cases where $(document).ready() is called after the + // browser event has already occurred. + if ( document.readyState === "complete" ) { + // Handle it asynchronously to allow scripts the opportunity to delay ready + return setTimeout( jQuery.ready, 1 ); + } + + // Mozilla, Opera and webkit nightlies currently support this event + if ( document.addEventListener ) { + // Use the handy event callback + document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false ); + + // A fallback to window.onload, that will always work + window.addEventListener( "load", jQuery.ready, false ); + + // If IE event model is used + } else if ( document.attachEvent ) { + // ensure firing before onload, + // maybe late but safe also for iframes + document.attachEvent( "onreadystatechange", DOMContentLoaded ); + + // A fallback to window.onload, that will always work + window.attachEvent( "onload", jQuery.ready ); + + // If IE and not a frame + // continually check to see if the document is ready + var toplevel = false; + + try { + toplevel = window.frameElement == null; + } catch(e) {} + + if ( document.documentElement.doScroll && toplevel ) { + doScrollCheck(); + } + } + }, + + // See test/unit/core.js for details concerning isFunction. + // Since version 1.3, DOM methods and functions like alert + // aren't supported. They return false on IE (#2968). + isFunction: function( obj ) { + return jQuery.type(obj) === "function"; + }, + + isArray: Array.isArray || function( obj ) { + return jQuery.type(obj) === "array"; + }, + + // A crude way of determining if an object is a window + isWindow: function( obj ) { + return obj && typeof obj === "object" && "setInterval" in obj; + }, + + isNaN: function( obj ) { + return obj == null || !rdigit.test( obj ) || isNaN( obj ); + }, + + type: function( obj ) { + return obj == null ? + String( obj ) : + class2type[ toString.call(obj) ] || "object"; + }, + + isPlainObject: function( obj ) { + // Must be an Object. + // Because of IE, we also have to check the presence of the constructor property. + // Make sure that DOM nodes and window objects don't pass through, as well + if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { + return false; + } + + try { + // Not own constructor property must be Object + if ( obj.constructor && + !hasOwn.call(obj, "constructor") && + !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { + return false; + } + } catch ( e ) { + // IE8,9 Will throw exceptions on certain host objects #9897 + return false; + } + + // Own properties are enumerated firstly, so to speed up, + // if last one is own, then all properties are own. + + var key; + for ( key in obj ) {} + + return key === undefined || hasOwn.call( obj, key ); + }, + + isEmptyObject: function( obj ) { + for ( var name in obj ) { + return false; + } + return true; + }, + + error: function( msg ) { + throw msg; + }, + + parseJSON: function( data ) { + if ( typeof data !== "string" || !data ) { + return null; + } + + // Make sure leading/trailing whitespace is removed (IE can't handle it) + data = jQuery.trim( data ); + + // Attempt to parse using the native JSON parser first + if ( window.JSON && window.JSON.parse ) { + return window.JSON.parse( data ); + } + + // Make sure the incoming data is actual JSON + // Logic borrowed from http://json.org/json2.js + if ( rvalidchars.test( data.replace( rvalidescape, "@" ) + .replace( rvalidtokens, "]" ) + .replace( rvalidbraces, "")) ) { + + return (new Function( "return " + data ))(); + + } + jQuery.error( "Invalid JSON: " + data ); + }, + + // Cross-browser xml parsing + parseXML: function( data ) { + var xml, tmp; + try { + if ( window.DOMParser ) { // Standard + tmp = new DOMParser(); + xml = tmp.parseFromString( data , "text/xml" ); + } else { // IE + xml = new ActiveXObject( "Microsoft.XMLDOM" ); + xml.async = "false"; + xml.loadXML( data ); + } + } catch( e ) { + xml = undefined; + } + if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) { + jQuery.error( "Invalid XML: " + data ); + } + return xml; + }, + + noop: function() {}, + + // Evaluates a script in a global context + // Workarounds based on findings by Jim Driscoll + // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context + globalEval: function( data ) { + if ( data && rnotwhite.test( data ) ) { + // We use execScript on Internet Explorer + // We use an anonymous function so that context is window + // rather than jQuery in Firefox + ( window.execScript || function( data ) { + window[ "eval" ].call( window, data ); + } )( data ); + } + }, + + // Convert dashed to camelCase; used by the css and data modules + // Microsoft forgot to hump their vendor prefix (#9572) + camelCase: function( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); + }, + + nodeName: function( elem, name ) { + return elem.nodeName && elem.nodeName.toUpperCase() === name.toUpperCase(); + }, + + // args is for internal usage only + each: function( object, callback, args ) { + var name, i = 0, + length = object.length, + isObj = length === undefined || jQuery.isFunction( object ); + + if ( args ) { + if ( isObj ) { + for ( name in object ) { + if ( callback.apply( object[ name ], args ) === false ) { + break; + } + } + } else { + for ( ; i < length; ) { + if ( callback.apply( object[ i++ ], args ) === false ) { + break; + } + } + } + + // A special, fast, case for the most common use of each + } else { + if ( isObj ) { + for ( name in object ) { + if ( callback.call( object[ name ], name, object[ name ] ) === false ) { + break; + } + } + } else { + for ( ; i < length; ) { + if ( callback.call( object[ i ], i, object[ i++ ] ) === false ) { + break; + } + } + } + } + + return object; + }, + + // Use native String.trim function wherever possible + trim: trim ? + function( text ) { + return text == null ? + "" : + trim.call( text ); + } : + + // Otherwise use our own trimming functionality + function( text ) { + return text == null ? + "" : + text.toString().replace( trimLeft, "" ).replace( trimRight, "" ); + }, + + // results is for internal usage only + makeArray: function( array, results ) { + var ret = results || []; + + if ( array != null ) { + // The window, strings (and functions) also have 'length' + // The extra typeof function check is to prevent crashes + // in Safari 2 (See: #3039) + // Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930 + var type = jQuery.type( array ); + + if ( array.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( array ) ) { + push.call( ret, array ); + } else { + jQuery.merge( ret, array ); + } + } + + return ret; + }, + + inArray: function( elem, array ) { + if ( !array ) { + return -1; + } + + if ( indexOf ) { + return indexOf.call( array, elem ); + } + + for ( var i = 0, length = array.length; i < length; i++ ) { + if ( array[ i ] === elem ) { + return i; + } + } + + return -1; + }, + + merge: function( first, second ) { + var i = first.length, + j = 0; + + if ( typeof second.length === "number" ) { + for ( var l = second.length; j < l; j++ ) { + first[ i++ ] = second[ j ]; + } + + } else { + while ( second[j] !== undefined ) { + first[ i++ ] = second[ j++ ]; + } + } + + first.length = i; + + return first; + }, + + grep: function( elems, callback, inv ) { + var ret = [], retVal; + inv = !!inv; + + // Go through the array, only saving the items + // that pass the validator function + for ( var i = 0, length = elems.length; i < length; i++ ) { + retVal = !!callback( elems[ i ], i ); + if ( inv !== retVal ) { + ret.push( elems[ i ] ); + } + } + + return ret; + }, + + // arg is for internal usage only + map: function( elems, callback, arg ) { + var value, key, ret = [], + i = 0, + length = elems.length, + // jquery objects are treated as arrays + isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ; + + // Go through the array, translating each of the items to their + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + + // Go through every key on the object, + } else { + for ( key in elems ) { + value = callback( elems[ key ], key, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + } + + // Flatten any nested arrays + return ret.concat.apply( [], ret ); + }, + + // A global GUID counter for objects + guid: 1, + + // Bind a function to a context, optionally partially applying any + // arguments. + proxy: function( fn, context ) { + if ( typeof context === "string" ) { + var tmp = fn[ context ]; + context = fn; + fn = tmp; + } + + // Quick check to determine if target is callable, in the spec + // this throws a TypeError, but we will just return undefined. + if ( !jQuery.isFunction( fn ) ) { + return undefined; + } + + // Simulated bind + var args = slice.call( arguments, 2 ), + proxy = function() { + return fn.apply( context, args.concat( slice.call( arguments ) ) ); + }; + + // Set the guid of unique handler to the same of original handler, so it can be removed + proxy.guid = fn.guid = fn.guid || proxy.guid || jQuery.guid++; + + return proxy; + }, + + // Mutifunctional method to get and set values to a collection + // The value/s can optionally be executed if it's a function + access: function( elems, key, value, exec, fn, pass ) { + var length = elems.length; + + // Setting many attributes + if ( typeof key === "object" ) { + for ( var k in key ) { + jQuery.access( elems, k, key[k], exec, fn, value ); + } + return elems; + } + + // Setting one attribute + if ( value !== undefined ) { + // Optionally, function values get executed if exec is true + exec = !pass && exec && jQuery.isFunction(value); + + for ( var i = 0; i < length; i++ ) { + fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass ); + } + + return elems; + } + + // Getting an attribute + return length ? fn( elems[0], key ) : undefined; + }, + + now: function() { + return (new Date()).getTime(); + }, + + // Use of jQuery.browser is frowned upon. + // More details: http://docs.jquery.com/Utilities/jQuery.browser + uaMatch: function( ua ) { + ua = ua.toLowerCase(); + + var match = rwebkit.exec( ua ) || + ropera.exec( ua ) || + rmsie.exec( ua ) || + ua.indexOf("compatible") < 0 && rmozilla.exec( ua ) || + []; + + return { browser: match[1] || "", version: match[2] || "0" }; + }, + + sub: function() { + function jQuerySub( selector, context ) { + return new jQuerySub.fn.init( selector, context ); + } + jQuery.extend( true, jQuerySub, this ); + jQuerySub.superclass = this; + jQuerySub.fn = jQuerySub.prototype = this(); + jQuerySub.fn.constructor = jQuerySub; + jQuerySub.sub = this.sub; + jQuerySub.fn.init = function init( selector, context ) { + if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) { + context = jQuerySub( context ); + } + + return jQuery.fn.init.call( this, selector, context, rootjQuerySub ); + }; + jQuerySub.fn.init.prototype = jQuerySub.fn; + var rootjQuerySub = jQuerySub(document); + return jQuerySub; + }, + + browser: {} +}); + +// Populate the class2type map +jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) { + class2type[ "[object " + name + "]" ] = name.toLowerCase(); +}); + +browserMatch = jQuery.uaMatch( userAgent ); +if ( browserMatch.browser ) { + jQuery.browser[ browserMatch.browser ] = true; + jQuery.browser.version = browserMatch.version; +} + +// Deprecated, use jQuery.browser.webkit instead +if ( jQuery.browser.webkit ) { + jQuery.browser.safari = true; +} + +// IE doesn't match non-breaking spaces with \s +if ( rnotwhite.test( "\xA0" ) ) { + trimLeft = /^[\s\xA0]+/; + trimRight = /[\s\xA0]+$/; +} + +// All jQuery objects should point back to these +rootjQuery = jQuery(document); + +// Cleanup functions for the document ready method +if ( document.addEventListener ) { + DOMContentLoaded = function() { + document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false ); + jQuery.ready(); + }; + +} else if ( document.attachEvent ) { + DOMContentLoaded = function() { + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( document.readyState === "complete" ) { + document.detachEvent( "onreadystatechange", DOMContentLoaded ); + jQuery.ready(); + } + }; +} + +// The DOM ready check for Internet Explorer +function doScrollCheck() { + if ( jQuery.isReady ) { + return; + } + + try { + // If IE is used, use the trick by Diego Perini + // http://javascript.nwbox.com/IEContentLoaded/ + document.documentElement.doScroll("left"); + } catch(e) { + setTimeout( doScrollCheck, 1 ); + return; + } + + // and execute any waiting functions + jQuery.ready(); +} + +return jQuery; + +})(); + + +var // Promise methods + promiseMethods = "done fail isResolved isRejected promise then always pipe".split( " " ), + // Static reference to slice + sliceDeferred = [].slice; + +jQuery.extend({ + // Create a simple deferred (one callbacks list) + _Deferred: function() { + var // callbacks list + callbacks = [], + // stored [ context , args ] + fired, + // to avoid firing when already doing so + firing, + // flag to know if the deferred has been cancelled + cancelled, + // the deferred itself + deferred = { + + // done( f1, f2, ...) + done: function() { + if ( !cancelled ) { + var args = arguments, + i, + length, + elem, + type, + _fired; + if ( fired ) { + _fired = fired; + fired = 0; + } + for ( i = 0, length = args.length; i < length; i++ ) { + elem = args[ i ]; + type = jQuery.type( elem ); + if ( type === "array" ) { + deferred.done.apply( deferred, elem ); + } else if ( type === "function" ) { + callbacks.push( elem ); + } + } + if ( _fired ) { + deferred.resolveWith( _fired[ 0 ], _fired[ 1 ] ); + } + } + return this; + }, + + // resolve with given context and args + resolveWith: function( context, args ) { + if ( !cancelled && !fired && !firing ) { + // make sure args are available (#8421) + args = args || []; + firing = 1; + try { + while( callbacks[ 0 ] ) { + callbacks.shift().apply( context, args ); + } + } + finally { + fired = [ context, args ]; + firing = 0; + } + } + return this; + }, + + // resolve with this as context and given arguments + resolve: function() { + deferred.resolveWith( this, arguments ); + return this; + }, + + // Has this deferred been resolved? + isResolved: function() { + return !!( firing || fired ); + }, + + // Cancel + cancel: function() { + cancelled = 1; + callbacks = []; + return this; + } + }; + + return deferred; + }, + + // Full fledged deferred (two callbacks list) + Deferred: function( func ) { + var deferred = jQuery._Deferred(), + failDeferred = jQuery._Deferred(), + promise; + // Add errorDeferred methods, then and promise + jQuery.extend( deferred, { + then: function( doneCallbacks, failCallbacks ) { + deferred.done( doneCallbacks ).fail( failCallbacks ); + return this; + }, + always: function() { + return deferred.done.apply( deferred, arguments ).fail.apply( this, arguments ); + }, + fail: failDeferred.done, + rejectWith: failDeferred.resolveWith, + reject: failDeferred.resolve, + isRejected: failDeferred.isResolved, + pipe: function( fnDone, fnFail ) { + return jQuery.Deferred(function( newDefer ) { + jQuery.each( { + done: [ fnDone, "resolve" ], + fail: [ fnFail, "reject" ] + }, function( handler, data ) { + var fn = data[ 0 ], + action = data[ 1 ], + returned; + if ( jQuery.isFunction( fn ) ) { + deferred[ handler ](function() { + returned = fn.apply( this, arguments ); + if ( returned && jQuery.isFunction( returned.promise ) ) { + returned.promise().then( newDefer.resolve, newDefer.reject ); + } else { + newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] ); + } + }); + } else { + deferred[ handler ]( newDefer[ action ] ); + } + }); + }).promise(); + }, + // Get a promise for this deferred + // If obj is provided, the promise aspect is added to the object + promise: function( obj ) { + if ( obj == null ) { + if ( promise ) { + return promise; + } + promise = obj = {}; + } + var i = promiseMethods.length; + while( i-- ) { + obj[ promiseMethods[i] ] = deferred[ promiseMethods[i] ]; + } + return obj; + } + }); + // Make sure only one callback list will be used + deferred.done( failDeferred.cancel ).fail( deferred.cancel ); + // Unexpose cancel + delete deferred.cancel; + // Call given func if any + if ( func ) { + func.call( deferred, deferred ); + } + return deferred; + }, + + // Deferred helper + when: function( firstParam ) { + var args = arguments, + i = 0, + length = args.length, + count = length, + deferred = length <= 1 && firstParam && jQuery.isFunction( firstParam.promise ) ? + firstParam : + jQuery.Deferred(); + function resolveFunc( i ) { + return function( value ) { + args[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value; + if ( !( --count ) ) { + // Strange bug in FF4: + // Values changed onto the arguments object sometimes end up as undefined values + // outside the $.when method. Cloning the object into a fresh array solves the issue + deferred.resolveWith( deferred, sliceDeferred.call( args, 0 ) ); + } + }; + } + if ( length > 1 ) { + for( ; i < length; i++ ) { + if ( args[ i ] && jQuery.isFunction( args[ i ].promise ) ) { + args[ i ].promise().then( resolveFunc(i), deferred.reject ); + } else { + --count; + } + } + if ( !count ) { + deferred.resolveWith( deferred, args ); + } + } else if ( deferred !== firstParam ) { + deferred.resolveWith( deferred, length ? [ firstParam ] : [] ); + } + return deferred.promise(); + } +}); + + + +jQuery.support = (function() { + + var div = document.createElement( "div" ), + documentElement = document.documentElement, + all, + a, + select, + opt, + input, + marginDiv, + support, + fragment, + body, + testElementParent, + testElement, + testElementStyle, + tds, + events, + eventName, + i, + isSupported; + + // Preliminary tests + div.setAttribute("className", "t"); + div.innerHTML = "
a"; + + + all = div.getElementsByTagName( "*" ); + a = div.getElementsByTagName( "a" )[ 0 ]; + + // Can't get basic test support + if ( !all || !all.length || !a ) { + return {}; + } + + // First batch of supports tests + select = document.createElement( "select" ); + opt = select.appendChild( document.createElement("option") ); + input = div.getElementsByTagName( "input" )[ 0 ]; + + support = { + // IE strips leading whitespace when .innerHTML is used + leadingWhitespace: ( div.firstChild.nodeType === 3 ), + + // Make sure that tbody elements aren't automatically inserted + // IE will insert them into empty tables + tbody: !div.getElementsByTagName( "tbody" ).length, + + // Make sure that link elements get serialized correctly by innerHTML + // This requires a wrapper element in IE + htmlSerialize: !!div.getElementsByTagName( "link" ).length, + + // Get the style information from getAttribute + // (IE uses .cssText instead) + style: /top/.test( a.getAttribute("style") ), + + // Make sure that URLs aren't manipulated + // (IE normalizes it by default) + hrefNormalized: ( a.getAttribute( "href" ) === "/a" ), + + // Make sure that element opacity exists + // (IE uses filter instead) + // Use a regex to work around a WebKit issue. See #5145 + opacity: /^0.55$/.test( a.style.opacity ), + + // Verify style float existence + // (IE uses styleFloat instead of cssFloat) + cssFloat: !!a.style.cssFloat, + + // Make sure that if no value is specified for a checkbox + // that it defaults to "on". + // (WebKit defaults to "" instead) + checkOn: ( input.value === "on" ), + + // Make sure that a selected-by-default option has a working selected property. + // (WebKit defaults to false instead of true, IE too, if it's in an optgroup) + optSelected: opt.selected, + + // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7) + getSetAttribute: div.className !== "t", + + // Will be defined later + submitBubbles: true, + changeBubbles: true, + focusinBubbles: false, + deleteExpando: true, + noCloneEvent: true, + inlineBlockNeedsLayout: false, + shrinkWrapBlocks: false, + reliableMarginRight: true + }; + + // Make sure checked status is properly cloned + input.checked = true; + support.noCloneChecked = input.cloneNode( true ).checked; + + // Make sure that the options inside disabled selects aren't marked as disabled + // (WebKit marks them as disabled) + select.disabled = true; + support.optDisabled = !opt.disabled; + + // Test to see if it's possible to delete an expando from an element + // Fails in Internet Explorer + try { + delete div.test; + } catch( e ) { + support.deleteExpando = false; + } + + if ( !div.addEventListener && div.attachEvent && div.fireEvent ) { + div.attachEvent( "onclick", function() { + // Cloning a node shouldn't copy over any + // bound event handlers (IE does this) + support.noCloneEvent = false; + }); + div.cloneNode( true ).fireEvent( "onclick" ); + } + + // Check if a radio maintains it's value + // after being appended to the DOM + input = document.createElement("input"); + input.value = "t"; + input.setAttribute("type", "radio"); + support.radioValue = input.value === "t"; + + input.setAttribute("checked", "checked"); + div.appendChild( input ); + fragment = document.createDocumentFragment(); + fragment.appendChild( div.firstChild ); + + // WebKit doesn't clone checked state correctly in fragments + support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked; + + div.innerHTML = ""; + + // Figure out if the W3C box model works as expected + div.style.width = div.style.paddingLeft = "1px"; + + body = document.getElementsByTagName( "body" )[ 0 ]; + // We use our own, invisible, body unless the body is already present + // in which case we use a div (#9239) + testElement = document.createElement( body ? "div" : "body" ); + testElementStyle = { + visibility: "hidden", + width: 0, + height: 0, + border: 0, + margin: 0, + background: "none" + }; + if ( body ) { + jQuery.extend( testElementStyle, { + position: "absolute", + left: "-1000px", + top: "-1000px" + }); + } + for ( i in testElementStyle ) { + testElement.style[ i ] = testElementStyle[ i ]; + } + testElement.appendChild( div ); + testElementParent = body || documentElement; + testElementParent.insertBefore( testElement, testElementParent.firstChild ); + + // Check if a disconnected checkbox will retain its checked + // value of true after appended to the DOM (IE6/7) + support.appendChecked = input.checked; + + support.boxModel = div.offsetWidth === 2; + + if ( "zoom" in div.style ) { + // Check if natively block-level elements act like inline-block + // elements when setting their display to 'inline' and giving + // them layout + // (IE < 8 does this) + div.style.display = "inline"; + div.style.zoom = 1; + support.inlineBlockNeedsLayout = ( div.offsetWidth === 2 ); + + // Check if elements with layout shrink-wrap their children + // (IE 6 does this) + div.style.display = ""; + div.innerHTML = "
"; + support.shrinkWrapBlocks = ( div.offsetWidth !== 2 ); + } + + div.innerHTML = "
t
"; + tds = div.getElementsByTagName( "td" ); + + // Check if table cells still have offsetWidth/Height when they are set + // to display:none and there are still other visible table cells in a + // table row; if so, offsetWidth/Height are not reliable for use when + // determining if an element has been hidden directly using + // display:none (it is still safe to use offsets if a parent element is + // hidden; don safety goggles and see bug #4512 for more information). + // (only IE 8 fails this test) + isSupported = ( tds[ 0 ].offsetHeight === 0 ); + + tds[ 0 ].style.display = ""; + tds[ 1 ].style.display = "none"; + + // Check if empty table cells still have offsetWidth/Height + // (IE < 8 fail this test) + support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 ); + div.innerHTML = ""; + + // Check if div with explicit width and no margin-right incorrectly + // gets computed margin-right based on width of container. For more + // info see bug #3333 + // Fails in WebKit before Feb 2011 nightlies + // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right + if ( document.defaultView && document.defaultView.getComputedStyle ) { + marginDiv = document.createElement( "div" ); + marginDiv.style.width = "0"; + marginDiv.style.marginRight = "0"; + div.appendChild( marginDiv ); + support.reliableMarginRight = + ( parseInt( ( document.defaultView.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0; + } + + // Remove the body element we added + testElement.innerHTML = ""; + testElementParent.removeChild( testElement ); + + // Technique from Juriy Zaytsev + // http://thinkweb2.com/projects/prototype/detecting-event-support-without-browser-sniffing/ + // We only care about the case where non-standard event systems + // are used, namely in IE. Short-circuiting here helps us to + // avoid an eval call (in setAttribute) which can cause CSP + // to go haywire. See: https://developer.mozilla.org/en/Security/CSP + if ( div.attachEvent ) { + for( i in { + submit: 1, + change: 1, + focusin: 1 + } ) { + eventName = "on" + i; + isSupported = ( eventName in div ); + if ( !isSupported ) { + div.setAttribute( eventName, "return;" ); + isSupported = ( typeof div[ eventName ] === "function" ); + } + support[ i + "Bubbles" ] = isSupported; + } + } + + // Null connected elements to avoid leaks in IE + testElement = fragment = select = opt = body = marginDiv = div = input = null; + + return support; +})(); + +// Keep track of boxModel +jQuery.boxModel = jQuery.support.boxModel; + + + + +var rbrace = /^(?:\{.*\}|\[.*\])$/, + rmultiDash = /([A-Z])/g; + +jQuery.extend({ + cache: {}, + + // Please use with caution + uuid: 0, + + // Unique for each copy of jQuery on the page + // Non-digits removed to match rinlinejQuery + expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ), + + // The following elements throw uncatchable exceptions if you + // attempt to add expando properties to them. + noData: { + "embed": true, + // Ban all objects except for Flash (which handle expandos) + "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000", + "applet": true + }, + + hasData: function( elem ) { + elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; + + return !!elem && !isEmptyDataObject( elem ); + }, + + data: function( elem, name, data, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, ret, + internalKey = jQuery.expando, + getByName = typeof name === "string", + + // We have to handle DOM nodes and JS objects differently because IE6-7 + // can't GC object references properly across the DOM-JS boundary + isNode = elem.nodeType, + + // Only DOM nodes need the global jQuery cache; JS object data is + // attached directly to the object so GC can occur automatically + cache = isNode ? jQuery.cache : elem, + + // Only defining an ID for JS objects if its cache already exists allows + // the code to shortcut on the same path as a DOM node with no cache + id = isNode ? elem[ jQuery.expando ] : elem[ jQuery.expando ] && jQuery.expando; + + // Avoid doing any more work than we need to when trying to get data on an + // object that has no data at all + if ( (!id || (pvt && id && (cache[ id ] && !cache[ id ][ internalKey ]))) && getByName && data === undefined ) { + return; + } + + if ( !id ) { + // Only DOM nodes need a new unique ID for each element since their data + // ends up in the global cache + if ( isNode ) { + elem[ jQuery.expando ] = id = ++jQuery.uuid; + } else { + id = jQuery.expando; + } + } + + if ( !cache[ id ] ) { + cache[ id ] = {}; + + // TODO: This is a hack for 1.5 ONLY. Avoids exposing jQuery + // metadata on plain JS objects when the object is serialized using + // JSON.stringify + if ( !isNode ) { + cache[ id ].toJSON = jQuery.noop; + } + } + + // An object can be passed to jQuery.data instead of a key/value pair; this gets + // shallow copied over onto the existing cache + if ( typeof name === "object" || typeof name === "function" ) { + if ( pvt ) { + cache[ id ][ internalKey ] = jQuery.extend(cache[ id ][ internalKey ], name); + } else { + cache[ id ] = jQuery.extend(cache[ id ], name); + } + } + + thisCache = cache[ id ]; + + // Internal jQuery data is stored in a separate object inside the object's data + // cache in order to avoid key collisions between internal data and user-defined + // data + if ( pvt ) { + if ( !thisCache[ internalKey ] ) { + thisCache[ internalKey ] = {}; + } + + thisCache = thisCache[ internalKey ]; + } + + if ( data !== undefined ) { + thisCache[ jQuery.camelCase( name ) ] = data; + } + + // TODO: This is a hack for 1.5 ONLY. It will be removed in 1.6. Users should + // not attempt to inspect the internal events object using jQuery.data, as this + // internal data object is undocumented and subject to change. + if ( name === "events" && !thisCache[name] ) { + return thisCache[ internalKey ] && thisCache[ internalKey ].events; + } + + // Check for both converted-to-camel and non-converted data property names + // If a data property was specified + if ( getByName ) { + + // First Try to find as-is property data + ret = thisCache[ name ]; + + // Test for null|undefined property data + if ( ret == null ) { + + // Try to find the camelCased property + ret = thisCache[ jQuery.camelCase( name ) ]; + } + } else { + ret = thisCache; + } + + return ret; + }, + + removeData: function( elem, name, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, + + // Reference to internal data cache key + internalKey = jQuery.expando, + + isNode = elem.nodeType, + + // See jQuery.data for more information + cache = isNode ? jQuery.cache : elem, + + // See jQuery.data for more information + id = isNode ? elem[ jQuery.expando ] : jQuery.expando; + + // If there is already no cache entry for this object, there is no + // purpose in continuing + if ( !cache[ id ] ) { + return; + } + + if ( name ) { + + thisCache = pvt ? cache[ id ][ internalKey ] : cache[ id ]; + + if ( thisCache ) { + + // Support interoperable removal of hyphenated or camelcased keys + if ( !thisCache[ name ] ) { + name = jQuery.camelCase( name ); + } + + delete thisCache[ name ]; + + // If there is no data left in the cache, we want to continue + // and let the cache object itself get destroyed + if ( !isEmptyDataObject(thisCache) ) { + return; + } + } + } + + // See jQuery.data for more information + if ( pvt ) { + delete cache[ id ][ internalKey ]; + + // Don't destroy the parent cache unless the internal data object + // had been the only thing left in it + if ( !isEmptyDataObject(cache[ id ]) ) { + return; + } + } + + var internalCache = cache[ id ][ internalKey ]; + + // Browsers that fail expando deletion also refuse to delete expandos on + // the window, but it will allow it on all other JS objects; other browsers + // don't care + // Ensure that `cache` is not a window object #10080 + if ( jQuery.support.deleteExpando || !cache.setInterval ) { + delete cache[ id ]; + } else { + cache[ id ] = null; + } + + // We destroyed the entire user cache at once because it's faster than + // iterating through each key, but we need to continue to persist internal + // data if it existed + if ( internalCache ) { + cache[ id ] = {}; + // TODO: This is a hack for 1.5 ONLY. Avoids exposing jQuery + // metadata on plain JS objects when the object is serialized using + // JSON.stringify + if ( !isNode ) { + cache[ id ].toJSON = jQuery.noop; + } + + cache[ id ][ internalKey ] = internalCache; + + // Otherwise, we need to eliminate the expando on the node to avoid + // false lookups in the cache for entries that no longer exist + } else if ( isNode ) { + // IE does not allow us to delete expando properties from nodes, + // nor does it have a removeAttribute function on Document nodes; + // we must handle all of these cases + if ( jQuery.support.deleteExpando ) { + delete elem[ jQuery.expando ]; + } else if ( elem.removeAttribute ) { + elem.removeAttribute( jQuery.expando ); + } else { + elem[ jQuery.expando ] = null; + } + } + }, + + // For internal use only. + _data: function( elem, name, data ) { + return jQuery.data( elem, name, data, true ); + }, + + // A method for determining if a DOM node can handle the data expando + acceptData: function( elem ) { + if ( elem.nodeName ) { + var match = jQuery.noData[ elem.nodeName.toLowerCase() ]; + + if ( match ) { + return !(match === true || elem.getAttribute("classid") !== match); + } + } + + return true; + } +}); + +jQuery.fn.extend({ + data: function( key, value ) { + var data = null; + + if ( typeof key === "undefined" ) { + if ( this.length ) { + data = jQuery.data( this[0] ); + + if ( this[0].nodeType === 1 ) { + var attr = this[0].attributes, name; + for ( var i = 0, l = attr.length; i < l; i++ ) { + name = attr[i].name; + + if ( name.indexOf( "data-" ) === 0 ) { + name = jQuery.camelCase( name.substring(5) ); + + dataAttr( this[0], name, data[ name ] ); + } + } + } + } + + return data; + + } else if ( typeof key === "object" ) { + return this.each(function() { + jQuery.data( this, key ); + }); + } + + var parts = key.split("."); + parts[1] = parts[1] ? "." + parts[1] : ""; + + if ( value === undefined ) { + data = this.triggerHandler("getData" + parts[1] + "!", [parts[0]]); + + // Try to fetch any internally stored data first + if ( data === undefined && this.length ) { + data = jQuery.data( this[0], key ); + data = dataAttr( this[0], key, data ); + } + + return data === undefined && parts[1] ? + this.data( parts[0] ) : + data; + + } else { + return this.each(function() { + var $this = jQuery( this ), + args = [ parts[0], value ]; + + $this.triggerHandler( "setData" + parts[1] + "!", args ); + jQuery.data( this, key, value ); + $this.triggerHandler( "changeData" + parts[1] + "!", args ); + }); + } + }, + + removeData: function( key ) { + return this.each(function() { + jQuery.removeData( this, key ); + }); + } +}); + +function dataAttr( elem, key, data ) { + // If nothing was found internally, try to fetch any + // data from the HTML5 data-* attribute + if ( data === undefined && elem.nodeType === 1 ) { + + var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); + + data = elem.getAttribute( name ); + + if ( typeof data === "string" ) { + try { + data = data === "true" ? true : + data === "false" ? false : + data === "null" ? null : + !jQuery.isNaN( data ) ? parseFloat( data ) : + rbrace.test( data ) ? jQuery.parseJSON( data ) : + data; + } catch( e ) {} + + // Make sure we set the data so it isn't changed later + jQuery.data( elem, key, data ); + + } else { + data = undefined; + } + } + + return data; +} + +// TODO: This is a hack for 1.5 ONLY to allow objects with a single toJSON +// property to be considered empty objects; this property always exists in +// order to make sure JSON.stringify does not expose internal metadata +function isEmptyDataObject( obj ) { + for ( var name in obj ) { + if ( name !== "toJSON" ) { + return false; + } + } + + return true; +} + + + + +function handleQueueMarkDefer( elem, type, src ) { + var deferDataKey = type + "defer", + queueDataKey = type + "queue", + markDataKey = type + "mark", + defer = jQuery.data( elem, deferDataKey, undefined, true ); + if ( defer && + ( src === "queue" || !jQuery.data( elem, queueDataKey, undefined, true ) ) && + ( src === "mark" || !jQuery.data( elem, markDataKey, undefined, true ) ) ) { + // Give room for hard-coded callbacks to fire first + // and eventually mark/queue something else on the element + setTimeout( function() { + if ( !jQuery.data( elem, queueDataKey, undefined, true ) && + !jQuery.data( elem, markDataKey, undefined, true ) ) { + jQuery.removeData( elem, deferDataKey, true ); + defer.resolve(); + } + }, 0 ); + } +} + +jQuery.extend({ + + _mark: function( elem, type ) { + if ( elem ) { + type = (type || "fx") + "mark"; + jQuery.data( elem, type, (jQuery.data(elem,type,undefined,true) || 0) + 1, true ); + } + }, + + _unmark: function( force, elem, type ) { + if ( force !== true ) { + type = elem; + elem = force; + force = false; + } + if ( elem ) { + type = type || "fx"; + var key = type + "mark", + count = force ? 0 : ( (jQuery.data( elem, key, undefined, true) || 1 ) - 1 ); + if ( count ) { + jQuery.data( elem, key, count, true ); + } else { + jQuery.removeData( elem, key, true ); + handleQueueMarkDefer( elem, type, "mark" ); + } + } + }, + + queue: function( elem, type, data ) { + if ( elem ) { + type = (type || "fx") + "queue"; + var q = jQuery.data( elem, type, undefined, true ); + // Speed up dequeue by getting out quickly if this is just a lookup + if ( data ) { + if ( !q || jQuery.isArray(data) ) { + q = jQuery.data( elem, type, jQuery.makeArray(data), true ); + } else { + q.push( data ); + } + } + return q || []; + } + }, + + dequeue: function( elem, type ) { + type = type || "fx"; + + var queue = jQuery.queue( elem, type ), + fn = queue.shift(), + defer; + + // If the fx queue is dequeued, always remove the progress sentinel + if ( fn === "inprogress" ) { + fn = queue.shift(); + } + + if ( fn ) { + // Add a progress sentinel to prevent the fx queue from being + // automatically dequeued + if ( type === "fx" ) { + queue.unshift("inprogress"); + } + + fn.call(elem, function() { + jQuery.dequeue(elem, type); + }); + } + + if ( !queue.length ) { + jQuery.removeData( elem, type + "queue", true ); + handleQueueMarkDefer( elem, type, "queue" ); + } + } +}); + +jQuery.fn.extend({ + queue: function( type, data ) { + if ( typeof type !== "string" ) { + data = type; + type = "fx"; + } + + if ( data === undefined ) { + return jQuery.queue( this[0], type ); + } + return this.each(function() { + var queue = jQuery.queue( this, type, data ); + + if ( type === "fx" && queue[0] !== "inprogress" ) { + jQuery.dequeue( this, type ); + } + }); + }, + dequeue: function( type ) { + return this.each(function() { + jQuery.dequeue( this, type ); + }); + }, + // Based off of the plugin by Clint Helfers, with permission. + // http://blindsignals.com/index.php/2009/07/jquery-delay/ + delay: function( time, type ) { + time = jQuery.fx ? jQuery.fx.speeds[time] || time : time; + type = type || "fx"; + + return this.queue( type, function() { + var elem = this; + setTimeout(function() { + jQuery.dequeue( elem, type ); + }, time ); + }); + }, + clearQueue: function( type ) { + return this.queue( type || "fx", [] ); + }, + // Get a promise resolved when queues of a certain type + // are emptied (fx is the type by default) + promise: function( type, object ) { + if ( typeof type !== "string" ) { + object = type; + type = undefined; + } + type = type || "fx"; + var defer = jQuery.Deferred(), + elements = this, + i = elements.length, + count = 1, + deferDataKey = type + "defer", + queueDataKey = type + "queue", + markDataKey = type + "mark", + tmp; + function resolve() { + if ( !( --count ) ) { + defer.resolveWith( elements, [ elements ] ); + } + } + while( i-- ) { + if (( tmp = jQuery.data( elements[ i ], deferDataKey, undefined, true ) || + ( jQuery.data( elements[ i ], queueDataKey, undefined, true ) || + jQuery.data( elements[ i ], markDataKey, undefined, true ) ) && + jQuery.data( elements[ i ], deferDataKey, jQuery._Deferred(), true ) )) { + count++; + tmp.done( resolve ); + } + } + resolve(); + return defer.promise(); + } +}); + + + + +var rclass = /[\n\t\r]/g, + rspace = /\s+/, + rreturn = /\r/g, + rtype = /^(?:button|input)$/i, + rfocusable = /^(?:button|input|object|select|textarea)$/i, + rclickable = /^a(?:rea)?$/i, + rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i, + nodeHook, boolHook; + +jQuery.fn.extend({ + attr: function( name, value ) { + return jQuery.access( this, name, value, true, jQuery.attr ); + }, + + removeAttr: function( name ) { + return this.each(function() { + jQuery.removeAttr( this, name ); + }); + }, + + prop: function( name, value ) { + return jQuery.access( this, name, value, true, jQuery.prop ); + }, + + removeProp: function( name ) { + name = jQuery.propFix[ name ] || name; + return this.each(function() { + // try/catch handles cases where IE balks (such as removing a property on window) + try { + this[ name ] = undefined; + delete this[ name ]; + } catch( e ) {} + }); + }, + + addClass: function( value ) { + var classNames, i, l, elem, + setClass, c, cl; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).addClass( value.call(this, j, this.className) ); + }); + } + + if ( value && typeof value === "string" ) { + classNames = value.split( rspace ); + + for ( i = 0, l = this.length; i < l; i++ ) { + elem = this[ i ]; + + if ( elem.nodeType === 1 ) { + if ( !elem.className && classNames.length === 1 ) { + elem.className = value; + + } else { + setClass = " " + elem.className + " "; + + for ( c = 0, cl = classNames.length; c < cl; c++ ) { + if ( !~setClass.indexOf( " " + classNames[ c ] + " " ) ) { + setClass += classNames[ c ] + " "; + } + } + elem.className = jQuery.trim( setClass ); + } + } + } + } + + return this; + }, + + removeClass: function( value ) { + var classNames, i, l, elem, className, c, cl; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).removeClass( value.call(this, j, this.className) ); + }); + } + + if ( (value && typeof value === "string") || value === undefined ) { + classNames = (value || "").split( rspace ); + + for ( i = 0, l = this.length; i < l; i++ ) { + elem = this[ i ]; + + if ( elem.nodeType === 1 && elem.className ) { + if ( value ) { + className = (" " + elem.className + " ").replace( rclass, " " ); + for ( c = 0, cl = classNames.length; c < cl; c++ ) { + className = className.replace(" " + classNames[ c ] + " ", " "); + } + elem.className = jQuery.trim( className ); + + } else { + elem.className = ""; + } + } + } + } + + return this; + }, + + toggleClass: function( value, stateVal ) { + var type = typeof value, + isBool = typeof stateVal === "boolean"; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( i ) { + jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal ); + }); + } + + return this.each(function() { + if ( type === "string" ) { + // toggle individual class names + var className, + i = 0, + self = jQuery( this ), + state = stateVal, + classNames = value.split( rspace ); + + while ( (className = classNames[ i++ ]) ) { + // check each className given, space seperated list + state = isBool ? state : !self.hasClass( className ); + self[ state ? "addClass" : "removeClass" ]( className ); + } + + } else if ( type === "undefined" || type === "boolean" ) { + if ( this.className ) { + // store className if set + jQuery._data( this, "__className__", this.className ); + } + + // toggle whole className + this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || ""; + } + }); + }, + + hasClass: function( selector ) { + var className = " " + selector + " "; + for ( var i = 0, l = this.length; i < l; i++ ) { + if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) { + return true; + } + } + + return false; + }, + + val: function( value ) { + var hooks, ret, + elem = this[0]; + + if ( !arguments.length ) { + if ( elem ) { + hooks = jQuery.valHooks[ elem.nodeName.toLowerCase() ] || jQuery.valHooks[ elem.type ]; + + if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) { + return ret; + } + + ret = elem.value; + + return typeof ret === "string" ? + // handle most common string cases + ret.replace(rreturn, "") : + // handle cases where value is null/undef or number + ret == null ? "" : ret; + } + + return undefined; + } + + var isFunction = jQuery.isFunction( value ); + + return this.each(function( i ) { + var self = jQuery(this), val; + + if ( this.nodeType !== 1 ) { + return; + } + + if ( isFunction ) { + val = value.call( this, i, self.val() ); + } else { + val = value; + } + + // Treat null/undefined as ""; convert numbers to string + if ( val == null ) { + val = ""; + } else if ( typeof val === "number" ) { + val += ""; + } else if ( jQuery.isArray( val ) ) { + val = jQuery.map(val, function ( value ) { + return value == null ? "" : value + ""; + }); + } + + hooks = jQuery.valHooks[ this.nodeName.toLowerCase() ] || jQuery.valHooks[ this.type ]; + + // If set returns undefined, fall back to normal setting + if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) { + this.value = val; + } + }); + } +}); + +jQuery.extend({ + valHooks: { + option: { + get: function( elem ) { + // attributes.value is undefined in Blackberry 4.7 but + // uses .value. See #6932 + var val = elem.attributes.value; + return !val || val.specified ? elem.value : elem.text; + } + }, + select: { + get: function( elem ) { + var value, + index = elem.selectedIndex, + values = [], + options = elem.options, + one = elem.type === "select-one"; + + // Nothing was selected + if ( index < 0 ) { + return null; + } + + // Loop through all the selected options + for ( var i = one ? index : 0, max = one ? index + 1 : options.length; i < max; i++ ) { + var option = options[ i ]; + + // Don't return options that are disabled or in a disabled optgroup + if ( option.selected && (jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null) && + (!option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" )) ) { + + // Get the specific value for the option + value = jQuery( option ).val(); + + // We don't need an array for one selects + if ( one ) { + return value; + } + + // Multi-Selects return an array + values.push( value ); + } + } + + // Fixes Bug #2551 -- select.val() broken in IE after form.reset() + if ( one && !values.length && options.length ) { + return jQuery( options[ index ] ).val(); + } + + return values; + }, + + set: function( elem, value ) { + var values = jQuery.makeArray( value ); + + jQuery(elem).find("option").each(function() { + this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0; + }); + + if ( !values.length ) { + elem.selectedIndex = -1; + } + return values; + } + } + }, + + attrFn: { + val: true, + css: true, + html: true, + text: true, + data: true, + width: true, + height: true, + offset: true + }, + + attrFix: { + // Always normalize to ensure hook usage + tabindex: "tabIndex" + }, + + attr: function( elem, name, value, pass ) { + var nType = elem.nodeType; + + // don't get/set attributes on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return undefined; + } + + if ( pass && name in jQuery.attrFn ) { + return jQuery( elem )[ name ]( value ); + } + + // Fallback to prop when attributes are not supported + if ( !("getAttribute" in elem) ) { + return jQuery.prop( elem, name, value ); + } + + var ret, hooks, + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + // Normalize the name if needed + if ( notxml ) { + name = jQuery.attrFix[ name ] || name; + + hooks = jQuery.attrHooks[ name ]; + + if ( !hooks ) { + // Use boolHook for boolean attributes + if ( rboolean.test( name ) ) { + hooks = boolHook; + + // Use nodeHook if available( IE6/7 ) + } else if ( nodeHook ) { + hooks = nodeHook; + } + } + } + + if ( value !== undefined ) { + + if ( value === null ) { + jQuery.removeAttr( elem, name ); + return undefined; + + } else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + elem.setAttribute( name, "" + value ); + return value; + } + + } else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + + ret = elem.getAttribute( name ); + + // Non-existent attributes return null, we normalize to undefined + return ret === null ? + undefined : + ret; + } + }, + + removeAttr: function( elem, name ) { + var propName; + if ( elem.nodeType === 1 ) { + name = jQuery.attrFix[ name ] || name; + + jQuery.attr( elem, name, "" ); + elem.removeAttribute( name ); + + // Set corresponding property to false for boolean attributes + if ( rboolean.test( name ) && (propName = jQuery.propFix[ name ] || name) in elem ) { + elem[ propName ] = false; + } + } + }, + + attrHooks: { + type: { + set: function( elem, value ) { + // We can't allow the type property to be changed (since it causes problems in IE) + if ( rtype.test( elem.nodeName ) && elem.parentNode ) { + jQuery.error( "type property can't be changed" ); + } else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) { + // Setting the type on a radio button after the value resets the value in IE6-9 + // Reset value to it's default in case type is set after value + // This is for element creation + var val = elem.value; + elem.setAttribute( "type", value ); + if ( val ) { + elem.value = val; + } + return value; + } + } + }, + // Use the value property for back compat + // Use the nodeHook for button elements in IE6/7 (#1954) + value: { + get: function( elem, name ) { + if ( nodeHook && jQuery.nodeName( elem, "button" ) ) { + return nodeHook.get( elem, name ); + } + return name in elem ? + elem.value : + null; + }, + set: function( elem, value, name ) { + if ( nodeHook && jQuery.nodeName( elem, "button" ) ) { + return nodeHook.set( elem, value, name ); + } + // Does not return so that setAttribute is also used + elem.value = value; + } + } + }, + + propFix: { + tabindex: "tabIndex", + readonly: "readOnly", + "for": "htmlFor", + "class": "className", + maxlength: "maxLength", + cellspacing: "cellSpacing", + cellpadding: "cellPadding", + rowspan: "rowSpan", + colspan: "colSpan", + usemap: "useMap", + frameborder: "frameBorder", + contenteditable: "contentEditable" + }, + + prop: function( elem, name, value ) { + var nType = elem.nodeType; + + // don't get/set properties on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return undefined; + } + + var ret, hooks, + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + if ( notxml ) { + // Fix name and attach hooks + name = jQuery.propFix[ name ] || name; + hooks = jQuery.propHooks[ name ]; + } + + if ( value !== undefined ) { + if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + return (elem[ name ] = value); + } + + } else { + if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + return elem[ name ]; + } + } + }, + + propHooks: { + tabIndex: { + get: function( elem ) { + // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set + // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ + var attributeNode = elem.getAttributeNode("tabindex"); + + return attributeNode && attributeNode.specified ? + parseInt( attributeNode.value, 10 ) : + rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ? + 0 : + undefined; + } + } + } +}); + +// Add the tabindex propHook to attrHooks for back-compat +jQuery.attrHooks.tabIndex = jQuery.propHooks.tabIndex; + +// Hook for boolean attributes +boolHook = { + get: function( elem, name ) { + // Align boolean attributes with corresponding properties + // Fall back to attribute presence where some booleans are not supported + var attrNode; + return jQuery.prop( elem, name ) === true || ( attrNode = elem.getAttributeNode( name ) ) && attrNode.nodeValue !== false ? + name.toLowerCase() : + undefined; + }, + set: function( elem, value, name ) { + var propName; + if ( value === false ) { + // Remove boolean attributes when set to false + jQuery.removeAttr( elem, name ); + } else { + // value is true since we know at this point it's type boolean and not false + // Set boolean attributes to the same name and set the DOM property + propName = jQuery.propFix[ name ] || name; + if ( propName in elem ) { + // Only set the IDL specifically if it already exists on the element + elem[ propName ] = true; + } + + elem.setAttribute( name, name.toLowerCase() ); + } + return name; + } +}; + +// IE6/7 do not support getting/setting some attributes with get/setAttribute +if ( !jQuery.support.getSetAttribute ) { + + // Use this for any attribute in IE6/7 + // This fixes almost every IE6/7 issue + nodeHook = jQuery.valHooks.button = { + get: function( elem, name ) { + var ret; + ret = elem.getAttributeNode( name ); + // Return undefined if nodeValue is empty string + return ret && ret.nodeValue !== "" ? + ret.nodeValue : + undefined; + }, + set: function( elem, value, name ) { + // Set the existing or create a new attribute node + var ret = elem.getAttributeNode( name ); + if ( !ret ) { + ret = document.createAttribute( name ); + elem.setAttributeNode( ret ); + } + return (ret.nodeValue = value + ""); + } + }; + + // Set width and height to auto instead of 0 on empty string( Bug #8150 ) + // This is for removals + jQuery.each([ "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], { + set: function( elem, value ) { + if ( value === "" ) { + elem.setAttribute( name, "auto" ); + return value; + } + } + }); + }); +} + + +// Some attributes require a special call on IE +if ( !jQuery.support.hrefNormalized ) { + jQuery.each([ "href", "src", "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], { + get: function( elem ) { + var ret = elem.getAttribute( name, 2 ); + return ret === null ? undefined : ret; + } + }); + }); +} + +if ( !jQuery.support.style ) { + jQuery.attrHooks.style = { + get: function( elem ) { + // Return undefined in the case of empty string + // Normalize to lowercase since IE uppercases css property names + return elem.style.cssText.toLowerCase() || undefined; + }, + set: function( elem, value ) { + return (elem.style.cssText = "" + value); + } + }; +} + +// Safari mis-reports the default selected property of an option +// Accessing the parent's selectedIndex property fixes it +if ( !jQuery.support.optSelected ) { + jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, { + get: function( elem ) { + var parent = elem.parentNode; + + if ( parent ) { + parent.selectedIndex; + + // Make sure that it also works with optgroups, see #5701 + if ( parent.parentNode ) { + parent.parentNode.selectedIndex; + } + } + return null; + } + }); +} + +// Radios and checkboxes getter/setter +if ( !jQuery.support.checkOn ) { + jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = { + get: function( elem ) { + // Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified + return elem.getAttribute("value") === null ? "on" : elem.value; + } + }; + }); +} +jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], { + set: function( elem, value ) { + if ( jQuery.isArray( value ) ) { + return (elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0); + } + } + }); +}); + + + + +var rnamespaces = /\.(.*)$/, + rformElems = /^(?:textarea|input|select)$/i, + rperiod = /\./g, + rspaces = / /g, + rescape = /[^\w\s.|`]/g, + fcleanup = function( nm ) { + return nm.replace(rescape, "\\$&"); + }; + +/* + * A number of helper functions used for managing events. + * Many of the ideas behind this code originated from + * Dean Edwards' addEvent library. + */ +jQuery.event = { + + // Bind an event to an element + // Original by Dean Edwards + add: function( elem, types, handler, data ) { + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + if ( handler === false ) { + handler = returnFalse; + } else if ( !handler ) { + // Fixes bug #7229. Fix recommended by jdalton + return; + } + + var handleObjIn, handleObj; + + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + } + + // Make sure that the function being executed has a unique ID + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure + var elemData = jQuery._data( elem ); + + // If no elemData is found then we must be trying to bind to one of the + // banned noData elements + if ( !elemData ) { + return; + } + + var events = elemData.events, + eventHandle = elemData.handle; + + if ( !events ) { + elemData.events = events = {}; + } + + if ( !eventHandle ) { + elemData.handle = eventHandle = function( e ) { + // Discard the second event of a jQuery.event.trigger() and + // when an event is called after a page has unloaded + return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ? + jQuery.event.handle.apply( eventHandle.elem, arguments ) : + undefined; + }; + } + + // Add elem as a property of the handle function + // This is to prevent a memory leak with non-native events in IE. + eventHandle.elem = elem; + + // Handle multiple events separated by a space + // jQuery(...).bind("mouseover mouseout", fn); + types = types.split(" "); + + var type, i = 0, namespaces; + + while ( (type = types[ i++ ]) ) { + handleObj = handleObjIn ? + jQuery.extend({}, handleObjIn) : + { handler: handler, data: data }; + + // Namespaced event handlers + if ( type.indexOf(".") > -1 ) { + namespaces = type.split("."); + type = namespaces.shift(); + handleObj.namespace = namespaces.slice(0).sort().join("."); + + } else { + namespaces = []; + handleObj.namespace = ""; + } + + handleObj.type = type; + if ( !handleObj.guid ) { + handleObj.guid = handler.guid; + } + + // Get the current list of functions bound to this event + var handlers = events[ type ], + special = jQuery.event.special[ type ] || {}; + + // Init the event handler queue + if ( !handlers ) { + handlers = events[ type ] = []; + + // Check for a special event handler + // Only use addEventListener/attachEvent if the special + // events handler returns false + if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + // Bind the global event handler to the element + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle, false ); + + } else if ( elem.attachEvent ) { + elem.attachEvent( "on" + type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add the function to the element's handler list + handlers.push( handleObj ); + + // Keep track of which events have been used, for event optimization + jQuery.event.global[ type ] = true; + } + + // Nullify elem to prevent memory leaks in IE + elem = null; + }, + + global: {}, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, pos ) { + // don't do events on text and comment nodes + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + if ( handler === false ) { + handler = returnFalse; + } + + var ret, type, fn, j, i = 0, all, namespaces, namespace, special, eventType, handleObj, origType, + elemData = jQuery.hasData( elem ) && jQuery._data( elem ), + events = elemData && elemData.events; + + if ( !elemData || !events ) { + return; + } + + // types is actually an event object here + if ( types && types.type ) { + handler = types.handler; + types = types.type; + } + + // Unbind all events for the element + if ( !types || typeof types === "string" && types.charAt(0) === "." ) { + types = types || ""; + + for ( type in events ) { + jQuery.event.remove( elem, type + types ); + } + + return; + } + + // Handle multiple events separated by a space + // jQuery(...).unbind("mouseover mouseout", fn); + types = types.split(" "); + + while ( (type = types[ i++ ]) ) { + origType = type; + handleObj = null; + all = type.indexOf(".") < 0; + namespaces = []; + + if ( !all ) { + // Namespaced event handlers + namespaces = type.split("."); + type = namespaces.shift(); + + namespace = new RegExp("(^|\\.)" + + jQuery.map( namespaces.slice(0).sort(), fcleanup ).join("\\.(?:.*\\.)?") + "(\\.|$)"); + } + + eventType = events[ type ]; + + if ( !eventType ) { + continue; + } + + if ( !handler ) { + for ( j = 0; j < eventType.length; j++ ) { + handleObj = eventType[ j ]; + + if ( all || namespace.test( handleObj.namespace ) ) { + jQuery.event.remove( elem, origType, handleObj.handler, j ); + eventType.splice( j--, 1 ); + } + } + + continue; + } + + special = jQuery.event.special[ type ] || {}; + + for ( j = pos || 0; j < eventType.length; j++ ) { + handleObj = eventType[ j ]; + + if ( handler.guid === handleObj.guid ) { + // remove the given handler for the given type + if ( all || namespace.test( handleObj.namespace ) ) { + if ( pos == null ) { + eventType.splice( j--, 1 ); + } + + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + + if ( pos != null ) { + break; + } + } + } + + // remove generic event handler if no more handlers exist + if ( eventType.length === 0 || pos != null && eventType.length === 1 ) { + if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) { + jQuery.removeEvent( elem, type, elemData.handle ); + } + + ret = null; + delete events[ type ]; + } + } + + // Remove the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + var handle = elemData.handle; + if ( handle ) { + handle.elem = null; + } + + delete elemData.events; + delete elemData.handle; + + if ( jQuery.isEmptyObject( elemData ) ) { + jQuery.removeData( elem, undefined, true ); + } + } + }, + + // Events that are safe to short-circuit if no handlers are attached. + // Native DOM events should not be added, they may have inline handlers. + customEvent: { + "getData": true, + "setData": true, + "changeData": true + }, + + trigger: function( event, data, elem, onlyHandlers ) { + // Event object or event type + var type = event.type || event, + namespaces = [], + exclusive; + + if ( type.indexOf("!") >= 0 ) { + // Exclusive events trigger only for the exact event (no namespaces) + type = type.slice(0, -1); + exclusive = true; + } + + if ( type.indexOf(".") >= 0 ) { + // Namespaced trigger; create a regexp to match event type in handle() + namespaces = type.split("."); + type = namespaces.shift(); + namespaces.sort(); + } + + if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) { + // No jQuery handlers for this event type, and it can't have inline handlers + return; + } + + // Caller can pass in an Event, Object, or just an event type string + event = typeof event === "object" ? + // jQuery.Event object + event[ jQuery.expando ] ? event : + // Object literal + new jQuery.Event( type, event ) : + // Just the event type (string) + new jQuery.Event( type ); + + event.type = type; + event.exclusive = exclusive; + event.namespace = namespaces.join("."); + event.namespace_re = new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)"); + + // triggerHandler() and global events don't bubble or run the default action + if ( onlyHandlers || !elem ) { + event.preventDefault(); + event.stopPropagation(); + } + + // Handle a global trigger + if ( !elem ) { + // TODO: Stop taunting the data cache; remove global events and always attach to document + jQuery.each( jQuery.cache, function() { + // internalKey variable is just used to make it easier to find + // and potentially change this stuff later; currently it just + // points to jQuery.expando + var internalKey = jQuery.expando, + internalCache = this[ internalKey ]; + if ( internalCache && internalCache.events && internalCache.events[ type ] ) { + jQuery.event.trigger( event, data, internalCache.handle.elem ); + } + }); + return; + } + + // Don't do events on text and comment nodes + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + // Clean up the event in case it is being reused + event.result = undefined; + event.target = elem; + + // Clone any incoming data and prepend the event, creating the handler arg list + data = data != null ? jQuery.makeArray( data ) : []; + data.unshift( event ); + + var cur = elem, + // IE doesn't like method names with a colon (#3533, #8272) + ontype = type.indexOf(":") < 0 ? "on" + type : ""; + + // Fire event on the current element, then bubble up the DOM tree + do { + var handle = jQuery._data( cur, "handle" ); + + event.currentTarget = cur; + if ( handle ) { + handle.apply( cur, data ); + } + + // Trigger an inline bound script + if ( ontype && jQuery.acceptData( cur ) && cur[ ontype ] && cur[ ontype ].apply( cur, data ) === false ) { + event.result = false; + event.preventDefault(); + } + + // Bubble up to document, then to window + cur = cur.parentNode || cur.ownerDocument || cur === event.target.ownerDocument && window; + } while ( cur && !event.isPropagationStopped() ); + + // If nobody prevented the default action, do it now + if ( !event.isDefaultPrevented() ) { + var old, + special = jQuery.event.special[ type ] || {}; + + if ( (!special._default || special._default.call( elem.ownerDocument, event ) === false) && + !(type === "click" && jQuery.nodeName( elem, "a" )) && jQuery.acceptData( elem ) ) { + + // Call a native DOM method on the target with the same name name as the event. + // Can't use an .isFunction)() check here because IE6/7 fails that test. + // IE<9 dies on focus to hidden element (#1486), may want to revisit a try/catch. + try { + if ( ontype && elem[ type ] ) { + // Don't re-trigger an onFOO event when we call its FOO() method + old = elem[ ontype ]; + + if ( old ) { + elem[ ontype ] = null; + } + + jQuery.event.triggered = type; + elem[ type ](); + } + } catch ( ieError ) {} + + if ( old ) { + elem[ ontype ] = old; + } + + jQuery.event.triggered = undefined; + } + } + + return event.result; + }, + + handle: function( event ) { + event = jQuery.event.fix( event || window.event ); + // Snapshot the handlers list since a called handler may add/remove events. + var handlers = ((jQuery._data( this, "events" ) || {})[ event.type ] || []).slice(0), + run_all = !event.exclusive && !event.namespace, + args = Array.prototype.slice.call( arguments, 0 ); + + // Use the fix-ed Event rather than the (read-only) native event + args[0] = event; + event.currentTarget = this; + + for ( var j = 0, l = handlers.length; j < l; j++ ) { + var handleObj = handlers[ j ]; + + // Triggered event must 1) be non-exclusive and have no namespace, or + // 2) have namespace(s) a subset or equal to those in the bound event. + if ( run_all || event.namespace_re.test( handleObj.namespace ) ) { + // Pass in a reference to the handler function itself + // So that we can later remove it + event.handler = handleObj.handler; + event.data = handleObj.data; + event.handleObj = handleObj; + + var ret = handleObj.handler.apply( this, args ); + + if ( ret !== undefined ) { + event.result = ret; + if ( ret === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + + if ( event.isImmediatePropagationStopped() ) { + break; + } + } + } + return event.result; + }, + + props: "altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "), + + fix: function( event ) { + if ( event[ jQuery.expando ] ) { + return event; + } + + // store a copy of the original event object + // and "clone" to set read-only properties + var originalEvent = event; + event = jQuery.Event( originalEvent ); + + for ( var i = this.props.length, prop; i; ) { + prop = this.props[ --i ]; + event[ prop ] = originalEvent[ prop ]; + } + + // Fix target property, if necessary + if ( !event.target ) { + // Fixes #1925 where srcElement might not be defined either + event.target = event.srcElement || document; + } + + // check if target is a textnode (safari) + if ( event.target.nodeType === 3 ) { + event.target = event.target.parentNode; + } + + // Add relatedTarget, if necessary + if ( !event.relatedTarget && event.fromElement ) { + event.relatedTarget = event.fromElement === event.target ? event.toElement : event.fromElement; + } + + // Calculate pageX/Y if missing and clientX/Y available + if ( event.pageX == null && event.clientX != null ) { + var eventDocument = event.target.ownerDocument || document, + doc = eventDocument.documentElement, + body = eventDocument.body; + + event.pageX = event.clientX + (doc && doc.scrollLeft || body && body.scrollLeft || 0) - (doc && doc.clientLeft || body && body.clientLeft || 0); + event.pageY = event.clientY + (doc && doc.scrollTop || body && body.scrollTop || 0) - (doc && doc.clientTop || body && body.clientTop || 0); + } + + // Add which for key events + if ( event.which == null && (event.charCode != null || event.keyCode != null) ) { + event.which = event.charCode != null ? event.charCode : event.keyCode; + } + + // Add metaKey to non-Mac browsers (use ctrl for PC's and Meta for Macs) + if ( !event.metaKey && event.ctrlKey ) { + event.metaKey = event.ctrlKey; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + // Note: button is not normalized, so don't use it + if ( !event.which && event.button !== undefined ) { + event.which = (event.button & 1 ? 1 : ( event.button & 2 ? 3 : ( event.button & 4 ? 2 : 0 ) )); + } + + return event; + }, + + // Deprecated, use jQuery.guid instead + guid: 1E8, + + // Deprecated, use jQuery.proxy instead + proxy: jQuery.proxy, + + special: { + ready: { + // Make sure the ready event is setup + setup: jQuery.bindReady, + teardown: jQuery.noop + }, + + live: { + add: function( handleObj ) { + jQuery.event.add( this, + liveConvert( handleObj.origType, handleObj.selector ), + jQuery.extend({}, handleObj, {handler: liveHandler, guid: handleObj.handler.guid}) ); + }, + + remove: function( handleObj ) { + jQuery.event.remove( this, liveConvert( handleObj.origType, handleObj.selector ), handleObj ); + } + }, + + beforeunload: { + setup: function( data, namespaces, eventHandle ) { + // We only want to do this special case on windows + if ( jQuery.isWindow( this ) ) { + this.onbeforeunload = eventHandle; + } + }, + + teardown: function( namespaces, eventHandle ) { + if ( this.onbeforeunload === eventHandle ) { + this.onbeforeunload = null; + } + } + } + } +}; + +jQuery.removeEvent = document.removeEventListener ? + function( elem, type, handle ) { + if ( elem.removeEventListener ) { + elem.removeEventListener( type, handle, false ); + } + } : + function( elem, type, handle ) { + if ( elem.detachEvent ) { + elem.detachEvent( "on" + type, handle ); + } + }; + +jQuery.Event = function( src, props ) { + // Allow instantiation without the 'new' keyword + if ( !this.preventDefault ) { + return new jQuery.Event( src, props ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + + // Events bubbling up the document may have been marked as prevented + // by a handler lower down the tree; reflect the correct value. + this.isDefaultPrevented = (src.defaultPrevented || src.returnValue === false || + src.getPreventDefault && src.getPreventDefault()) ? returnTrue : returnFalse; + + // Event type + } else { + this.type = src; + } + + // Put explicitly provided properties onto the event object + if ( props ) { + jQuery.extend( this, props ); + } + + // timeStamp is buggy for some events on Firefox(#3843) + // So we won't rely on the native value + this.timeStamp = jQuery.now(); + + // Mark it as fixed + this[ jQuery.expando ] = true; +}; + +function returnFalse() { + return false; +} +function returnTrue() { + return true; +} + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + preventDefault: function() { + this.isDefaultPrevented = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + + // if preventDefault exists run it on the original event + if ( e.preventDefault ) { + e.preventDefault(); + + // otherwise set the returnValue property of the original event to false (IE) + } else { + e.returnValue = false; + } + }, + stopPropagation: function() { + this.isPropagationStopped = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + // if stopPropagation exists run it on the original event + if ( e.stopPropagation ) { + e.stopPropagation(); + } + // otherwise set the cancelBubble property of the original event to true (IE) + e.cancelBubble = true; + }, + stopImmediatePropagation: function() { + this.isImmediatePropagationStopped = returnTrue; + this.stopPropagation(); + }, + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse +}; + +// Checks if an event happened on an element within another element +// Used in jQuery.event.special.mouseenter and mouseleave handlers +var withinElement = function( event ) { + + // Check if mouse(over|out) are still within the same parent element + var related = event.relatedTarget, + inside = false, + eventType = event.type; + + event.type = event.data; + + if ( related !== this ) { + + if ( related ) { + inside = jQuery.contains( this, related ); + } + + if ( !inside ) { + + jQuery.event.handle.apply( this, arguments ); + + event.type = eventType; + } + } +}, + +// In case of event delegation, we only need to rename the event.type, +// liveHandler will take care of the rest. +delegate = function( event ) { + event.type = event.data; + jQuery.event.handle.apply( this, arguments ); +}; + +// Create mouseenter and mouseleave events +jQuery.each({ + mouseenter: "mouseover", + mouseleave: "mouseout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + setup: function( data ) { + jQuery.event.add( this, fix, data && data.selector ? delegate : withinElement, orig ); + }, + teardown: function( data ) { + jQuery.event.remove( this, fix, data && data.selector ? delegate : withinElement ); + } + }; +}); + +// submit delegation +if ( !jQuery.support.submitBubbles ) { + + jQuery.event.special.submit = { + setup: function( data, namespaces ) { + if ( !jQuery.nodeName( this, "form" ) ) { + jQuery.event.add(this, "click.specialSubmit", function( e ) { + // Avoid triggering error on non-existent type attribute in IE VML (#7071) + var elem = e.target, + type = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.type : ""; + + if ( (type === "submit" || type === "image") && jQuery( elem ).closest("form").length ) { + trigger( "submit", this, arguments ); + } + }); + + jQuery.event.add(this, "keypress.specialSubmit", function( e ) { + var elem = e.target, + type = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.type : ""; + + if ( (type === "text" || type === "password") && jQuery( elem ).closest("form").length && e.keyCode === 13 ) { + trigger( "submit", this, arguments ); + } + }); + + } else { + return false; + } + }, + + teardown: function( namespaces ) { + jQuery.event.remove( this, ".specialSubmit" ); + } + }; + +} + +// change delegation, happens here so we have bind. +if ( !jQuery.support.changeBubbles ) { + + var changeFilters, + + getVal = function( elem ) { + var type = jQuery.nodeName( elem, "input" ) ? elem.type : "", + val = elem.value; + + if ( type === "radio" || type === "checkbox" ) { + val = elem.checked; + + } else if ( type === "select-multiple" ) { + val = elem.selectedIndex > -1 ? + jQuery.map( elem.options, function( elem ) { + return elem.selected; + }).join("-") : + ""; + + } else if ( jQuery.nodeName( elem, "select" ) ) { + val = elem.selectedIndex; + } + + return val; + }, + + testChange = function testChange( e ) { + var elem = e.target, data, val; + + if ( !rformElems.test( elem.nodeName ) || elem.readOnly ) { + return; + } + + data = jQuery._data( elem, "_change_data" ); + val = getVal(elem); + + // the current data will be also retrieved by beforeactivate + if ( e.type !== "focusout" || elem.type !== "radio" ) { + jQuery._data( elem, "_change_data", val ); + } + + if ( data === undefined || val === data ) { + return; + } + + if ( data != null || val ) { + e.type = "change"; + e.liveFired = undefined; + jQuery.event.trigger( e, arguments[1], elem ); + } + }; + + jQuery.event.special.change = { + filters: { + focusout: testChange, + + beforedeactivate: testChange, + + click: function( e ) { + var elem = e.target, type = jQuery.nodeName( elem, "input" ) ? elem.type : ""; + + if ( type === "radio" || type === "checkbox" || jQuery.nodeName( elem, "select" ) ) { + testChange.call( this, e ); + } + }, + + // Change has to be called before submit + // Keydown will be called before keypress, which is used in submit-event delegation + keydown: function( e ) { + var elem = e.target, type = jQuery.nodeName( elem, "input" ) ? elem.type : ""; + + if ( (e.keyCode === 13 && !jQuery.nodeName( elem, "textarea" ) ) || + (e.keyCode === 32 && (type === "checkbox" || type === "radio")) || + type === "select-multiple" ) { + testChange.call( this, e ); + } + }, + + // Beforeactivate happens also before the previous element is blurred + // with this event you can't trigger a change event, but you can store + // information + beforeactivate: function( e ) { + var elem = e.target; + jQuery._data( elem, "_change_data", getVal(elem) ); + } + }, + + setup: function( data, namespaces ) { + if ( this.type === "file" ) { + return false; + } + + for ( var type in changeFilters ) { + jQuery.event.add( this, type + ".specialChange", changeFilters[type] ); + } + + return rformElems.test( this.nodeName ); + }, + + teardown: function( namespaces ) { + jQuery.event.remove( this, ".specialChange" ); + + return rformElems.test( this.nodeName ); + } + }; + + changeFilters = jQuery.event.special.change.filters; + + // Handle when the input is .focus()'d + changeFilters.focus = changeFilters.beforeactivate; +} + +function trigger( type, elem, args ) { + // Piggyback on a donor event to simulate a different one. + // Fake originalEvent to avoid donor's stopPropagation, but if the + // simulated event prevents default then we do the same on the donor. + // Don't pass args or remember liveFired; they apply to the donor event. + var event = jQuery.extend( {}, args[ 0 ] ); + event.type = type; + event.originalEvent = {}; + event.liveFired = undefined; + jQuery.event.handle.call( elem, event ); + if ( event.isDefaultPrevented() ) { + args[ 0 ].preventDefault(); + } +} + +// Create "bubbling" focus and blur events +if ( !jQuery.support.focusinBubbles ) { + jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { + + // Attach a single capturing handler while someone wants focusin/focusout + var attaches = 0; + + jQuery.event.special[ fix ] = { + setup: function() { + if ( attaches++ === 0 ) { + document.addEventListener( orig, handler, true ); + } + }, + teardown: function() { + if ( --attaches === 0 ) { + document.removeEventListener( orig, handler, true ); + } + } + }; + + function handler( donor ) { + // Donor event is always a native one; fix it and switch its type. + // Let focusin/out handler cancel the donor focus/blur event. + var e = jQuery.event.fix( donor ); + e.type = fix; + e.originalEvent = {}; + jQuery.event.trigger( e, null, e.target ); + if ( e.isDefaultPrevented() ) { + donor.preventDefault(); + } + } + }); +} + +jQuery.each(["bind", "one"], function( i, name ) { + jQuery.fn[ name ] = function( type, data, fn ) { + var handler; + + // Handle object literals + if ( typeof type === "object" ) { + for ( var key in type ) { + this[ name ](key, data, type[key], fn); + } + return this; + } + + if ( arguments.length === 2 || data === false ) { + fn = data; + data = undefined; + } + + if ( name === "one" ) { + handler = function( event ) { + jQuery( this ).unbind( event, handler ); + return fn.apply( this, arguments ); + }; + handler.guid = fn.guid || jQuery.guid++; + } else { + handler = fn; + } + + if ( type === "unload" && name !== "one" ) { + this.one( type, data, fn ); + + } else { + for ( var i = 0, l = this.length; i < l; i++ ) { + jQuery.event.add( this[i], type, handler, data ); + } + } + + return this; + }; +}); + +jQuery.fn.extend({ + unbind: function( type, fn ) { + // Handle object literals + if ( typeof type === "object" && !type.preventDefault ) { + for ( var key in type ) { + this.unbind(key, type[key]); + } + + } else { + for ( var i = 0, l = this.length; i < l; i++ ) { + jQuery.event.remove( this[i], type, fn ); + } + } + + return this; + }, + + delegate: function( selector, types, data, fn ) { + return this.live( types, data, fn, selector ); + }, + + undelegate: function( selector, types, fn ) { + if ( arguments.length === 0 ) { + return this.unbind( "live" ); + + } else { + return this.die( types, null, fn, selector ); + } + }, + + trigger: function( type, data ) { + return this.each(function() { + jQuery.event.trigger( type, data, this ); + }); + }, + + triggerHandler: function( type, data ) { + if ( this[0] ) { + return jQuery.event.trigger( type, data, this[0], true ); + } + }, + + toggle: function( fn ) { + // Save reference to arguments for access in closure + var args = arguments, + guid = fn.guid || jQuery.guid++, + i = 0, + toggler = function( event ) { + // Figure out which function to execute + var lastToggle = ( jQuery.data( this, "lastToggle" + fn.guid ) || 0 ) % i; + jQuery.data( this, "lastToggle" + fn.guid, lastToggle + 1 ); + + // Make sure that clicks stop + event.preventDefault(); + + // and execute the function + return args[ lastToggle ].apply( this, arguments ) || false; + }; + + // link all the functions, so any of them can unbind this click handler + toggler.guid = guid; + while ( i < args.length ) { + args[ i++ ].guid = guid; + } + + return this.click( toggler ); + }, + + hover: function( fnOver, fnOut ) { + return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver ); + } +}); + +var liveMap = { + focus: "focusin", + blur: "focusout", + mouseenter: "mouseover", + mouseleave: "mouseout" +}; + +jQuery.each(["live", "die"], function( i, name ) { + jQuery.fn[ name ] = function( types, data, fn, origSelector /* Internal Use Only */ ) { + var type, i = 0, match, namespaces, preType, + selector = origSelector || this.selector, + context = origSelector ? this : jQuery( this.context ); + + if ( typeof types === "object" && !types.preventDefault ) { + for ( var key in types ) { + context[ name ]( key, data, types[key], selector ); + } + + return this; + } + + if ( name === "die" && !types && + origSelector && origSelector.charAt(0) === "." ) { + + context.unbind( origSelector ); + + return this; + } + + if ( data === false || jQuery.isFunction( data ) ) { + fn = data || returnFalse; + data = undefined; + } + + types = (types || "").split(" "); + + while ( (type = types[ i++ ]) != null ) { + match = rnamespaces.exec( type ); + namespaces = ""; + + if ( match ) { + namespaces = match[0]; + type = type.replace( rnamespaces, "" ); + } + + if ( type === "hover" ) { + types.push( "mouseenter" + namespaces, "mouseleave" + namespaces ); + continue; + } + + preType = type; + + if ( liveMap[ type ] ) { + types.push( liveMap[ type ] + namespaces ); + type = type + namespaces; + + } else { + type = (liveMap[ type ] || type) + namespaces; + } + + if ( name === "live" ) { + // bind live handler + for ( var j = 0, l = context.length; j < l; j++ ) { + jQuery.event.add( context[j], "live." + liveConvert( type, selector ), + { data: data, selector: selector, handler: fn, origType: type, origHandler: fn, preType: preType } ); + } + + } else { + // unbind live handler + context.unbind( "live." + liveConvert( type, selector ), fn ); + } + } + + return this; + }; +}); + +function liveHandler( event ) { + var stop, maxLevel, related, match, handleObj, elem, j, i, l, data, close, namespace, ret, + elems = [], + selectors = [], + events = jQuery._data( this, "events" ); + + // Make sure we avoid non-left-click bubbling in Firefox (#3861) and disabled elements in IE (#6911) + if ( event.liveFired === this || !events || !events.live || event.target.disabled || event.button && event.type === "click" ) { + return; + } + + if ( event.namespace ) { + namespace = new RegExp("(^|\\.)" + event.namespace.split(".").join("\\.(?:.*\\.)?") + "(\\.|$)"); + } + + event.liveFired = this; + + var live = events.live.slice(0); + + for ( j = 0; j < live.length; j++ ) { + handleObj = live[j]; + + if ( handleObj.origType.replace( rnamespaces, "" ) === event.type ) { + selectors.push( handleObj.selector ); + + } else { + live.splice( j--, 1 ); + } + } + + match = jQuery( event.target ).closest( selectors, event.currentTarget ); + + for ( i = 0, l = match.length; i < l; i++ ) { + close = match[i]; + + for ( j = 0; j < live.length; j++ ) { + handleObj = live[j]; + + if ( close.selector === handleObj.selector && (!namespace || namespace.test( handleObj.namespace )) && !close.elem.disabled ) { + elem = close.elem; + related = null; + + // Those two events require additional checking + if ( handleObj.preType === "mouseenter" || handleObj.preType === "mouseleave" ) { + event.type = handleObj.preType; + related = jQuery( event.relatedTarget ).closest( handleObj.selector )[0]; + + // Make sure not to accidentally match a child element with the same selector + if ( related && jQuery.contains( elem, related ) ) { + related = elem; + } + } + + if ( !related || related !== elem ) { + elems.push({ elem: elem, handleObj: handleObj, level: close.level }); + } + } + } + } + + for ( i = 0, l = elems.length; i < l; i++ ) { + match = elems[i]; + + if ( maxLevel && match.level > maxLevel ) { + break; + } + + event.currentTarget = match.elem; + event.data = match.handleObj.data; + event.handleObj = match.handleObj; + + ret = match.handleObj.origHandler.apply( match.elem, arguments ); + + if ( ret === false || event.isPropagationStopped() ) { + maxLevel = match.level; + + if ( ret === false ) { + stop = false; + } + if ( event.isImmediatePropagationStopped() ) { + break; + } + } + } + + return stop; +} + +function liveConvert( type, selector ) { + return (type && type !== "*" ? type + "." : "") + selector.replace(rperiod, "`").replace(rspaces, "&"); +} + +jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " + + "mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " + + "change select submit keydown keypress keyup error").split(" "), function( i, name ) { + + // Handle event binding + jQuery.fn[ name ] = function( data, fn ) { + if ( fn == null ) { + fn = data; + data = null; + } + + return arguments.length > 0 ? + this.bind( name, data, fn ) : + this.trigger( name ); + }; + + if ( jQuery.attrFn ) { + jQuery.attrFn[ name ] = true; + } +}); + + + +/*! + * Sizzle CSS Selector Engine + * Copyright 2011, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * More information: http://sizzlejs.com/ + */ +(function(){ + +var chunker = /((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g, + done = 0, + toString = Object.prototype.toString, + hasDuplicate = false, + baseHasDuplicate = true, + rBackslash = /\\/g, + rNonWord = /\W/; + +// Here we check if the JavaScript engine is using some sort of +// optimization where it does not always call our comparision +// function. If that is the case, discard the hasDuplicate value. +// Thus far that includes Google Chrome. +[0, 0].sort(function() { + baseHasDuplicate = false; + return 0; +}); + +var Sizzle = function( selector, context, results, seed ) { + results = results || []; + context = context || document; + + var origContext = context; + + if ( context.nodeType !== 1 && context.nodeType !== 9 ) { + return []; + } + + if ( !selector || typeof selector !== "string" ) { + return results; + } + + var m, set, checkSet, extra, ret, cur, pop, i, + prune = true, + contextXML = Sizzle.isXML( context ), + parts = [], + soFar = selector; + + // Reset the position of the chunker regexp (start from head) + do { + chunker.exec( "" ); + m = chunker.exec( soFar ); + + if ( m ) { + soFar = m[3]; + + parts.push( m[1] ); + + if ( m[2] ) { + extra = m[3]; + break; + } + } + } while ( m ); + + if ( parts.length > 1 && origPOS.exec( selector ) ) { + + if ( parts.length === 2 && Expr.relative[ parts[0] ] ) { + set = posProcess( parts[0] + parts[1], context ); + + } else { + set = Expr.relative[ parts[0] ] ? + [ context ] : + Sizzle( parts.shift(), context ); + + while ( parts.length ) { + selector = parts.shift(); + + if ( Expr.relative[ selector ] ) { + selector += parts.shift(); + } + + set = posProcess( selector, set ); + } + } + + } else { + // Take a shortcut and set the context if the root selector is an ID + // (but not if it'll be faster if the inner selector is an ID) + if ( !seed && parts.length > 1 && context.nodeType === 9 && !contextXML && + Expr.match.ID.test(parts[0]) && !Expr.match.ID.test(parts[parts.length - 1]) ) { + + ret = Sizzle.find( parts.shift(), context, contextXML ); + context = ret.expr ? + Sizzle.filter( ret.expr, ret.set )[0] : + ret.set[0]; + } + + if ( context ) { + ret = seed ? + { expr: parts.pop(), set: makeArray(seed) } : + Sizzle.find( parts.pop(), parts.length === 1 && (parts[0] === "~" || parts[0] === "+") && context.parentNode ? context.parentNode : context, contextXML ); + + set = ret.expr ? + Sizzle.filter( ret.expr, ret.set ) : + ret.set; + + if ( parts.length > 0 ) { + checkSet = makeArray( set ); + + } else { + prune = false; + } + + while ( parts.length ) { + cur = parts.pop(); + pop = cur; + + if ( !Expr.relative[ cur ] ) { + cur = ""; + } else { + pop = parts.pop(); + } + + if ( pop == null ) { + pop = context; + } + + Expr.relative[ cur ]( checkSet, pop, contextXML ); + } + + } else { + checkSet = parts = []; + } + } + + if ( !checkSet ) { + checkSet = set; + } + + if ( !checkSet ) { + Sizzle.error( cur || selector ); + } + + if ( toString.call(checkSet) === "[object Array]" ) { + if ( !prune ) { + results.push.apply( results, checkSet ); + + } else if ( context && context.nodeType === 1 ) { + for ( i = 0; checkSet[i] != null; i++ ) { + if ( checkSet[i] && (checkSet[i] === true || checkSet[i].nodeType === 1 && Sizzle.contains(context, checkSet[i])) ) { + results.push( set[i] ); + } + } + + } else { + for ( i = 0; checkSet[i] != null; i++ ) { + if ( checkSet[i] && checkSet[i].nodeType === 1 ) { + results.push( set[i] ); + } + } + } + + } else { + makeArray( checkSet, results ); + } + + if ( extra ) { + Sizzle( extra, origContext, results, seed ); + Sizzle.uniqueSort( results ); + } + + return results; +}; + +Sizzle.uniqueSort = function( results ) { + if ( sortOrder ) { + hasDuplicate = baseHasDuplicate; + results.sort( sortOrder ); + + if ( hasDuplicate ) { + for ( var i = 1; i < results.length; i++ ) { + if ( results[i] === results[ i - 1 ] ) { + results.splice( i--, 1 ); + } + } + } + } + + return results; +}; + +Sizzle.matches = function( expr, set ) { + return Sizzle( expr, null, null, set ); +}; + +Sizzle.matchesSelector = function( node, expr ) { + return Sizzle( expr, null, null, [node] ).length > 0; +}; + +Sizzle.find = function( expr, context, isXML ) { + var set; + + if ( !expr ) { + return []; + } + + for ( var i = 0, l = Expr.order.length; i < l; i++ ) { + var match, + type = Expr.order[i]; + + if ( (match = Expr.leftMatch[ type ].exec( expr )) ) { + var left = match[1]; + match.splice( 1, 1 ); + + if ( left.substr( left.length - 1 ) !== "\\" ) { + match[1] = (match[1] || "").replace( rBackslash, "" ); + set = Expr.find[ type ]( match, context, isXML ); + + if ( set != null ) { + expr = expr.replace( Expr.match[ type ], "" ); + break; + } + } + } + } + + if ( !set ) { + set = typeof context.getElementsByTagName !== "undefined" ? + context.getElementsByTagName( "*" ) : + []; + } + + return { set: set, expr: expr }; +}; + +Sizzle.filter = function( expr, set, inplace, not ) { + var match, anyFound, + old = expr, + result = [], + curLoop = set, + isXMLFilter = set && set[0] && Sizzle.isXML( set[0] ); + + while ( expr && set.length ) { + for ( var type in Expr.filter ) { + if ( (match = Expr.leftMatch[ type ].exec( expr )) != null && match[2] ) { + var found, item, + filter = Expr.filter[ type ], + left = match[1]; + + anyFound = false; + + match.splice(1,1); + + if ( left.substr( left.length - 1 ) === "\\" ) { + continue; + } + + if ( curLoop === result ) { + result = []; + } + + if ( Expr.preFilter[ type ] ) { + match = Expr.preFilter[ type ]( match, curLoop, inplace, result, not, isXMLFilter ); + + if ( !match ) { + anyFound = found = true; + + } else if ( match === true ) { + continue; + } + } + + if ( match ) { + for ( var i = 0; (item = curLoop[i]) != null; i++ ) { + if ( item ) { + found = filter( item, match, i, curLoop ); + var pass = not ^ !!found; + + if ( inplace && found != null ) { + if ( pass ) { + anyFound = true; + + } else { + curLoop[i] = false; + } + + } else if ( pass ) { + result.push( item ); + anyFound = true; + } + } + } + } + + if ( found !== undefined ) { + if ( !inplace ) { + curLoop = result; + } + + expr = expr.replace( Expr.match[ type ], "" ); + + if ( !anyFound ) { + return []; + } + + break; + } + } + } + + // Improper expression + if ( expr === old ) { + if ( anyFound == null ) { + Sizzle.error( expr ); + + } else { + break; + } + } + + old = expr; + } + + return curLoop; +}; + +Sizzle.error = function( msg ) { + throw "Syntax error, unrecognized expression: " + msg; +}; + +var Expr = Sizzle.selectors = { + order: [ "ID", "NAME", "TAG" ], + + match: { + ID: /#((?:[\w\u00c0-\uFFFF\-]|\\.)+)/, + CLASS: /\.((?:[\w\u00c0-\uFFFF\-]|\\.)+)/, + NAME: /\[name=['"]*((?:[\w\u00c0-\uFFFF\-]|\\.)+)['"]*\]/, + ATTR: /\[\s*((?:[\w\u00c0-\uFFFF\-]|\\.)+)\s*(?:(\S?=)\s*(?:(['"])(.*?)\3|(#?(?:[\w\u00c0-\uFFFF\-]|\\.)*)|)|)\s*\]/, + TAG: /^((?:[\w\u00c0-\uFFFF\*\-]|\\.)+)/, + CHILD: /:(only|nth|last|first)-child(?:\(\s*(even|odd|(?:[+\-]?\d+|(?:[+\-]?\d*)?n\s*(?:[+\-]\s*\d+)?))\s*\))?/, + POS: /:(nth|eq|gt|lt|first|last|even|odd)(?:\((\d*)\))?(?=[^\-]|$)/, + PSEUDO: /:((?:[\w\u00c0-\uFFFF\-]|\\.)+)(?:\((['"]?)((?:\([^\)]+\)|[^\(\)]*)+)\2\))?/ + }, + + leftMatch: {}, + + attrMap: { + "class": "className", + "for": "htmlFor" + }, + + attrHandle: { + href: function( elem ) { + return elem.getAttribute( "href" ); + }, + type: function( elem ) { + return elem.getAttribute( "type" ); + } + }, + + relative: { + "+": function(checkSet, part){ + var isPartStr = typeof part === "string", + isTag = isPartStr && !rNonWord.test( part ), + isPartStrNotTag = isPartStr && !isTag; + + if ( isTag ) { + part = part.toLowerCase(); + } + + for ( var i = 0, l = checkSet.length, elem; i < l; i++ ) { + if ( (elem = checkSet[i]) ) { + while ( (elem = elem.previousSibling) && elem.nodeType !== 1 ) {} + + checkSet[i] = isPartStrNotTag || elem && elem.nodeName.toLowerCase() === part ? + elem || false : + elem === part; + } + } + + if ( isPartStrNotTag ) { + Sizzle.filter( part, checkSet, true ); + } + }, + + ">": function( checkSet, part ) { + var elem, + isPartStr = typeof part === "string", + i = 0, + l = checkSet.length; + + if ( isPartStr && !rNonWord.test( part ) ) { + part = part.toLowerCase(); + + for ( ; i < l; i++ ) { + elem = checkSet[i]; + + if ( elem ) { + var parent = elem.parentNode; + checkSet[i] = parent.nodeName.toLowerCase() === part ? parent : false; + } + } + + } else { + for ( ; i < l; i++ ) { + elem = checkSet[i]; + + if ( elem ) { + checkSet[i] = isPartStr ? + elem.parentNode : + elem.parentNode === part; + } + } + + if ( isPartStr ) { + Sizzle.filter( part, checkSet, true ); + } + } + }, + + "": function(checkSet, part, isXML){ + var nodeCheck, + doneName = done++, + checkFn = dirCheck; + + if ( typeof part === "string" && !rNonWord.test( part ) ) { + part = part.toLowerCase(); + nodeCheck = part; + checkFn = dirNodeCheck; + } + + checkFn( "parentNode", part, doneName, checkSet, nodeCheck, isXML ); + }, + + "~": function( checkSet, part, isXML ) { + var nodeCheck, + doneName = done++, + checkFn = dirCheck; + + if ( typeof part === "string" && !rNonWord.test( part ) ) { + part = part.toLowerCase(); + nodeCheck = part; + checkFn = dirNodeCheck; + } + + checkFn( "previousSibling", part, doneName, checkSet, nodeCheck, isXML ); + } + }, + + find: { + ID: function( match, context, isXML ) { + if ( typeof context.getElementById !== "undefined" && !isXML ) { + var m = context.getElementById(match[1]); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + return m && m.parentNode ? [m] : []; + } + }, + + NAME: function( match, context ) { + if ( typeof context.getElementsByName !== "undefined" ) { + var ret = [], + results = context.getElementsByName( match[1] ); + + for ( var i = 0, l = results.length; i < l; i++ ) { + if ( results[i].getAttribute("name") === match[1] ) { + ret.push( results[i] ); + } + } + + return ret.length === 0 ? null : ret; + } + }, + + TAG: function( match, context ) { + if ( typeof context.getElementsByTagName !== "undefined" ) { + return context.getElementsByTagName( match[1] ); + } + } + }, + preFilter: { + CLASS: function( match, curLoop, inplace, result, not, isXML ) { + match = " " + match[1].replace( rBackslash, "" ) + " "; + + if ( isXML ) { + return match; + } + + for ( var i = 0, elem; (elem = curLoop[i]) != null; i++ ) { + if ( elem ) { + if ( not ^ (elem.className && (" " + elem.className + " ").replace(/[\t\n\r]/g, " ").indexOf(match) >= 0) ) { + if ( !inplace ) { + result.push( elem ); + } + + } else if ( inplace ) { + curLoop[i] = false; + } + } + } + + return false; + }, + + ID: function( match ) { + return match[1].replace( rBackslash, "" ); + }, + + TAG: function( match, curLoop ) { + return match[1].replace( rBackslash, "" ).toLowerCase(); + }, + + CHILD: function( match ) { + if ( match[1] === "nth" ) { + if ( !match[2] ) { + Sizzle.error( match[0] ); + } + + match[2] = match[2].replace(/^\+|\s*/g, ''); + + // parse equations like 'even', 'odd', '5', '2n', '3n+2', '4n-1', '-n+6' + var test = /(-?)(\d*)(?:n([+\-]?\d*))?/.exec( + match[2] === "even" && "2n" || match[2] === "odd" && "2n+1" || + !/\D/.test( match[2] ) && "0n+" + match[2] || match[2]); + + // calculate the numbers (first)n+(last) including if they are negative + match[2] = (test[1] + (test[2] || 1)) - 0; + match[3] = test[3] - 0; + } + else if ( match[2] ) { + Sizzle.error( match[0] ); + } + + // TODO: Move to normal caching system + match[0] = done++; + + return match; + }, + + ATTR: function( match, curLoop, inplace, result, not, isXML ) { + var name = match[1] = match[1].replace( rBackslash, "" ); + + if ( !isXML && Expr.attrMap[name] ) { + match[1] = Expr.attrMap[name]; + } + + // Handle if an un-quoted value was used + match[4] = ( match[4] || match[5] || "" ).replace( rBackslash, "" ); + + if ( match[2] === "~=" ) { + match[4] = " " + match[4] + " "; + } + + return match; + }, + + PSEUDO: function( match, curLoop, inplace, result, not ) { + if ( match[1] === "not" ) { + // If we're dealing with a complex expression, or a simple one + if ( ( chunker.exec(match[3]) || "" ).length > 1 || /^\w/.test(match[3]) ) { + match[3] = Sizzle(match[3], null, null, curLoop); + + } else { + var ret = Sizzle.filter(match[3], curLoop, inplace, true ^ not); + + if ( !inplace ) { + result.push.apply( result, ret ); + } + + return false; + } + + } else if ( Expr.match.POS.test( match[0] ) || Expr.match.CHILD.test( match[0] ) ) { + return true; + } + + return match; + }, + + POS: function( match ) { + match.unshift( true ); + + return match; + } + }, + + filters: { + enabled: function( elem ) { + return elem.disabled === false && elem.type !== "hidden"; + }, + + disabled: function( elem ) { + return elem.disabled === true; + }, + + checked: function( elem ) { + return elem.checked === true; + }, + + selected: function( elem ) { + // Accessing this property makes selected-by-default + // options in Safari work properly + if ( elem.parentNode ) { + elem.parentNode.selectedIndex; + } + + return elem.selected === true; + }, + + parent: function( elem ) { + return !!elem.firstChild; + }, + + empty: function( elem ) { + return !elem.firstChild; + }, + + has: function( elem, i, match ) { + return !!Sizzle( match[3], elem ).length; + }, + + header: function( elem ) { + return (/h\d/i).test( elem.nodeName ); + }, + + text: function( elem ) { + var attr = elem.getAttribute( "type" ), type = elem.type; + // IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) + // use getAttribute instead to test this case + return elem.nodeName.toLowerCase() === "input" && "text" === type && ( attr === type || attr === null ); + }, + + radio: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "radio" === elem.type; + }, + + checkbox: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "checkbox" === elem.type; + }, + + file: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "file" === elem.type; + }, + + password: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "password" === elem.type; + }, + + submit: function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && "submit" === elem.type; + }, + + image: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "image" === elem.type; + }, + + reset: function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && "reset" === elem.type; + }, + + button: function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && "button" === elem.type || name === "button"; + }, + + input: function( elem ) { + return (/input|select|textarea|button/i).test( elem.nodeName ); + }, + + focus: function( elem ) { + return elem === elem.ownerDocument.activeElement; + } + }, + setFilters: { + first: function( elem, i ) { + return i === 0; + }, + + last: function( elem, i, match, array ) { + return i === array.length - 1; + }, + + even: function( elem, i ) { + return i % 2 === 0; + }, + + odd: function( elem, i ) { + return i % 2 === 1; + }, + + lt: function( elem, i, match ) { + return i < match[3] - 0; + }, + + gt: function( elem, i, match ) { + return i > match[3] - 0; + }, + + nth: function( elem, i, match ) { + return match[3] - 0 === i; + }, + + eq: function( elem, i, match ) { + return match[3] - 0 === i; + } + }, + filter: { + PSEUDO: function( elem, match, i, array ) { + var name = match[1], + filter = Expr.filters[ name ]; + + if ( filter ) { + return filter( elem, i, match, array ); + + } else if ( name === "contains" ) { + return (elem.textContent || elem.innerText || Sizzle.getText([ elem ]) || "").indexOf(match[3]) >= 0; + + } else if ( name === "not" ) { + var not = match[3]; + + for ( var j = 0, l = not.length; j < l; j++ ) { + if ( not[j] === elem ) { + return false; + } + } + + return true; + + } else { + Sizzle.error( name ); + } + }, + + CHILD: function( elem, match ) { + var type = match[1], + node = elem; + + switch ( type ) { + case "only": + case "first": + while ( (node = node.previousSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + + if ( type === "first" ) { + return true; + } + + node = elem; + + case "last": + while ( (node = node.nextSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + + return true; + + case "nth": + var first = match[2], + last = match[3]; + + if ( first === 1 && last === 0 ) { + return true; + } + + var doneName = match[0], + parent = elem.parentNode; + + if ( parent && (parent.sizcache !== doneName || !elem.nodeIndex) ) { + var count = 0; + + for ( node = parent.firstChild; node; node = node.nextSibling ) { + if ( node.nodeType === 1 ) { + node.nodeIndex = ++count; + } + } + + parent.sizcache = doneName; + } + + var diff = elem.nodeIndex - last; + + if ( first === 0 ) { + return diff === 0; + + } else { + return ( diff % first === 0 && diff / first >= 0 ); + } + } + }, + + ID: function( elem, match ) { + return elem.nodeType === 1 && elem.getAttribute("id") === match; + }, + + TAG: function( elem, match ) { + return (match === "*" && elem.nodeType === 1) || elem.nodeName.toLowerCase() === match; + }, + + CLASS: function( elem, match ) { + return (" " + (elem.className || elem.getAttribute("class")) + " ") + .indexOf( match ) > -1; + }, + + ATTR: function( elem, match ) { + var name = match[1], + result = Expr.attrHandle[ name ] ? + Expr.attrHandle[ name ]( elem ) : + elem[ name ] != null ? + elem[ name ] : + elem.getAttribute( name ), + value = result + "", + type = match[2], + check = match[4]; + + return result == null ? + type === "!=" : + type === "=" ? + value === check : + type === "*=" ? + value.indexOf(check) >= 0 : + type === "~=" ? + (" " + value + " ").indexOf(check) >= 0 : + !check ? + value && result !== false : + type === "!=" ? + value !== check : + type === "^=" ? + value.indexOf(check) === 0 : + type === "$=" ? + value.substr(value.length - check.length) === check : + type === "|=" ? + value === check || value.substr(0, check.length + 1) === check + "-" : + false; + }, + + POS: function( elem, match, i, array ) { + var name = match[2], + filter = Expr.setFilters[ name ]; + + if ( filter ) { + return filter( elem, i, match, array ); + } + } + } +}; + +var origPOS = Expr.match.POS, + fescape = function(all, num){ + return "\\" + (num - 0 + 1); + }; + +for ( var type in Expr.match ) { + Expr.match[ type ] = new RegExp( Expr.match[ type ].source + (/(?![^\[]*\])(?![^\(]*\))/.source) ); + Expr.leftMatch[ type ] = new RegExp( /(^(?:.|\r|\n)*?)/.source + Expr.match[ type ].source.replace(/\\(\d+)/g, fescape) ); +} + +var makeArray = function( array, results ) { + array = Array.prototype.slice.call( array, 0 ); + + if ( results ) { + results.push.apply( results, array ); + return results; + } + + return array; +}; + +// Perform a simple check to determine if the browser is capable of +// converting a NodeList to an array using builtin methods. +// Also verifies that the returned array holds DOM nodes +// (which is not the case in the Blackberry browser) +try { + Array.prototype.slice.call( document.documentElement.childNodes, 0 )[0].nodeType; + +// Provide a fallback method if it does not work +} catch( e ) { + makeArray = function( array, results ) { + var i = 0, + ret = results || []; + + if ( toString.call(array) === "[object Array]" ) { + Array.prototype.push.apply( ret, array ); + + } else { + if ( typeof array.length === "number" ) { + for ( var l = array.length; i < l; i++ ) { + ret.push( array[i] ); + } + + } else { + for ( ; array[i]; i++ ) { + ret.push( array[i] ); + } + } + } + + return ret; + }; +} + +var sortOrder, siblingCheck; + +if ( document.documentElement.compareDocumentPosition ) { + sortOrder = function( a, b ) { + if ( a === b ) { + hasDuplicate = true; + return 0; + } + + if ( !a.compareDocumentPosition || !b.compareDocumentPosition ) { + return a.compareDocumentPosition ? -1 : 1; + } + + return a.compareDocumentPosition(b) & 4 ? -1 : 1; + }; + +} else { + sortOrder = function( a, b ) { + // The nodes are identical, we can exit early + if ( a === b ) { + hasDuplicate = true; + return 0; + + // Fallback to using sourceIndex (in IE) if it's available on both nodes + } else if ( a.sourceIndex && b.sourceIndex ) { + return a.sourceIndex - b.sourceIndex; + } + + var al, bl, + ap = [], + bp = [], + aup = a.parentNode, + bup = b.parentNode, + cur = aup; + + // If the nodes are siblings (or identical) we can do a quick check + if ( aup === bup ) { + return siblingCheck( a, b ); + + // If no parents were found then the nodes are disconnected + } else if ( !aup ) { + return -1; + + } else if ( !bup ) { + return 1; + } + + // Otherwise they're somewhere else in the tree so we need + // to build up a full list of the parentNodes for comparison + while ( cur ) { + ap.unshift( cur ); + cur = cur.parentNode; + } + + cur = bup; + + while ( cur ) { + bp.unshift( cur ); + cur = cur.parentNode; + } + + al = ap.length; + bl = bp.length; + + // Start walking down the tree looking for a discrepancy + for ( var i = 0; i < al && i < bl; i++ ) { + if ( ap[i] !== bp[i] ) { + return siblingCheck( ap[i], bp[i] ); + } + } + + // We ended someplace up the tree so do a sibling check + return i === al ? + siblingCheck( a, bp[i], -1 ) : + siblingCheck( ap[i], b, 1 ); + }; + + siblingCheck = function( a, b, ret ) { + if ( a === b ) { + return ret; + } + + var cur = a.nextSibling; + + while ( cur ) { + if ( cur === b ) { + return -1; + } + + cur = cur.nextSibling; + } + + return 1; + }; +} + +// Utility function for retreiving the text value of an array of DOM nodes +Sizzle.getText = function( elems ) { + var ret = "", elem; + + for ( var i = 0; elems[i]; i++ ) { + elem = elems[i]; + + // Get the text from text nodes and CDATA nodes + if ( elem.nodeType === 3 || elem.nodeType === 4 ) { + ret += elem.nodeValue; + + // Traverse everything else, except comment nodes + } else if ( elem.nodeType !== 8 ) { + ret += Sizzle.getText( elem.childNodes ); + } + } + + return ret; +}; + +// Check to see if the browser returns elements by name when +// querying by getElementById (and provide a workaround) +(function(){ + // We're going to inject a fake input element with a specified name + var form = document.createElement("div"), + id = "script" + (new Date()).getTime(), + root = document.documentElement; + + form.innerHTML = ""; + + // Inject it into the root element, check its status, and remove it quickly + root.insertBefore( form, root.firstChild ); + + // The workaround has to do additional checks after a getElementById + // Which slows things down for other browsers (hence the branching) + if ( document.getElementById( id ) ) { + Expr.find.ID = function( match, context, isXML ) { + if ( typeof context.getElementById !== "undefined" && !isXML ) { + var m = context.getElementById(match[1]); + + return m ? + m.id === match[1] || typeof m.getAttributeNode !== "undefined" && m.getAttributeNode("id").nodeValue === match[1] ? + [m] : + undefined : + []; + } + }; + + Expr.filter.ID = function( elem, match ) { + var node = typeof elem.getAttributeNode !== "undefined" && elem.getAttributeNode("id"); + + return elem.nodeType === 1 && node && node.nodeValue === match; + }; + } + + root.removeChild( form ); + + // release memory in IE + root = form = null; +})(); + +(function(){ + // Check to see if the browser returns only elements + // when doing getElementsByTagName("*") + + // Create a fake element + var div = document.createElement("div"); + div.appendChild( document.createComment("") ); + + // Make sure no comments are found + if ( div.getElementsByTagName("*").length > 0 ) { + Expr.find.TAG = function( match, context ) { + var results = context.getElementsByTagName( match[1] ); + + // Filter out possible comments + if ( match[1] === "*" ) { + var tmp = []; + + for ( var i = 0; results[i]; i++ ) { + if ( results[i].nodeType === 1 ) { + tmp.push( results[i] ); + } + } + + results = tmp; + } + + return results; + }; + } + + // Check to see if an attribute returns normalized href attributes + div.innerHTML = ""; + + if ( div.firstChild && typeof div.firstChild.getAttribute !== "undefined" && + div.firstChild.getAttribute("href") !== "#" ) { + + Expr.attrHandle.href = function( elem ) { + return elem.getAttribute( "href", 2 ); + }; + } + + // release memory in IE + div = null; +})(); + +if ( document.querySelectorAll ) { + (function(){ + var oldSizzle = Sizzle, + div = document.createElement("div"), + id = "__sizzle__"; + + div.innerHTML = "

"; + + // Safari can't handle uppercase or unicode characters when + // in quirks mode. + if ( div.querySelectorAll && div.querySelectorAll(".TEST").length === 0 ) { + return; + } + + Sizzle = function( query, context, extra, seed ) { + context = context || document; + + // Only use querySelectorAll on non-XML documents + // (ID selectors don't work in non-HTML documents) + if ( !seed && !Sizzle.isXML(context) ) { + // See if we find a selector to speed up + var match = /^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec( query ); + + if ( match && (context.nodeType === 1 || context.nodeType === 9) ) { + // Speed-up: Sizzle("TAG") + if ( match[1] ) { + return makeArray( context.getElementsByTagName( query ), extra ); + + // Speed-up: Sizzle(".CLASS") + } else if ( match[2] && Expr.find.CLASS && context.getElementsByClassName ) { + return makeArray( context.getElementsByClassName( match[2] ), extra ); + } + } + + if ( context.nodeType === 9 ) { + // Speed-up: Sizzle("body") + // The body element only exists once, optimize finding it + if ( query === "body" && context.body ) { + return makeArray( [ context.body ], extra ); + + // Speed-up: Sizzle("#ID") + } else if ( match && match[3] ) { + var elem = context.getElementById( match[3] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id === match[3] ) { + return makeArray( [ elem ], extra ); + } + + } else { + return makeArray( [], extra ); + } + } + + try { + return makeArray( context.querySelectorAll(query), extra ); + } catch(qsaError) {} + + // qSA works strangely on Element-rooted queries + // We can work around this by specifying an extra ID on the root + // and working up from there (Thanks to Andrew Dupont for the technique) + // IE 8 doesn't work on object elements + } else if ( context.nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { + var oldContext = context, + old = context.getAttribute( "id" ), + nid = old || id, + hasParent = context.parentNode, + relativeHierarchySelector = /^\s*[+~]/.test( query ); + + if ( !old ) { + context.setAttribute( "id", nid ); + } else { + nid = nid.replace( /'/g, "\\$&" ); + } + if ( relativeHierarchySelector && hasParent ) { + context = context.parentNode; + } + + try { + if ( !relativeHierarchySelector || hasParent ) { + return makeArray( context.querySelectorAll( "[id='" + nid + "'] " + query ), extra ); + } + + } catch(pseudoError) { + } finally { + if ( !old ) { + oldContext.removeAttribute( "id" ); + } + } + } + } + + return oldSizzle(query, context, extra, seed); + }; + + for ( var prop in oldSizzle ) { + Sizzle[ prop ] = oldSizzle[ prop ]; + } + + // release memory in IE + div = null; + })(); +} + +(function(){ + var html = document.documentElement, + matches = html.matchesSelector || html.mozMatchesSelector || html.webkitMatchesSelector || html.msMatchesSelector; + + if ( matches ) { + // Check to see if it's possible to do matchesSelector + // on a disconnected node (IE 9 fails this) + var disconnectedMatch = !matches.call( document.createElement( "div" ), "div" ), + pseudoWorks = false; + + try { + // This should fail with an exception + // Gecko does not error, returns false instead + matches.call( document.documentElement, "[test!='']:sizzle" ); + + } catch( pseudoError ) { + pseudoWorks = true; + } + + Sizzle.matchesSelector = function( node, expr ) { + // Make sure that attribute selectors are quoted + expr = expr.replace(/\=\s*([^'"\]]*)\s*\]/g, "='$1']"); + + if ( !Sizzle.isXML( node ) ) { + try { + if ( pseudoWorks || !Expr.match.PSEUDO.test( expr ) && !/!=/.test( expr ) ) { + var ret = matches.call( node, expr ); + + // IE 9's matchesSelector returns false on disconnected nodes + if ( ret || !disconnectedMatch || + // As well, disconnected nodes are said to be in a document + // fragment in IE 9, so check for that + node.document && node.document.nodeType !== 11 ) { + return ret; + } + } + } catch(e) {} + } + + return Sizzle(expr, null, null, [node]).length > 0; + }; + } +})(); + +(function(){ + var div = document.createElement("div"); + + div.innerHTML = "
"; + + // Opera can't find a second classname (in 9.6) + // Also, make sure that getElementsByClassName actually exists + if ( !div.getElementsByClassName || div.getElementsByClassName("e").length === 0 ) { + return; + } + + // Safari caches class attributes, doesn't catch changes (in 3.2) + div.lastChild.className = "e"; + + if ( div.getElementsByClassName("e").length === 1 ) { + return; + } + + Expr.order.splice(1, 0, "CLASS"); + Expr.find.CLASS = function( match, context, isXML ) { + if ( typeof context.getElementsByClassName !== "undefined" && !isXML ) { + return context.getElementsByClassName(match[1]); + } + }; + + // release memory in IE + div = null; +})(); + +function dirNodeCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + + if ( elem ) { + var match = false; + + elem = elem[dir]; + + while ( elem ) { + if ( elem.sizcache === doneName ) { + match = checkSet[elem.sizset]; + break; + } + + if ( elem.nodeType === 1 && !isXML ){ + elem.sizcache = doneName; + elem.sizset = i; + } + + if ( elem.nodeName.toLowerCase() === cur ) { + match = elem; + break; + } + + elem = elem[dir]; + } + + checkSet[i] = match; + } + } +} + +function dirCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + + if ( elem ) { + var match = false; + + elem = elem[dir]; + + while ( elem ) { + if ( elem.sizcache === doneName ) { + match = checkSet[elem.sizset]; + break; + } + + if ( elem.nodeType === 1 ) { + if ( !isXML ) { + elem.sizcache = doneName; + elem.sizset = i; + } + + if ( typeof cur !== "string" ) { + if ( elem === cur ) { + match = true; + break; + } + + } else if ( Sizzle.filter( cur, [elem] ).length > 0 ) { + match = elem; + break; + } + } + + elem = elem[dir]; + } + + checkSet[i] = match; + } + } +} + +if ( document.documentElement.contains ) { + Sizzle.contains = function( a, b ) { + return a !== b && (a.contains ? a.contains(b) : true); + }; + +} else if ( document.documentElement.compareDocumentPosition ) { + Sizzle.contains = function( a, b ) { + return !!(a.compareDocumentPosition(b) & 16); + }; + +} else { + Sizzle.contains = function() { + return false; + }; +} + +Sizzle.isXML = function( elem ) { + // documentElement is verified for cases where it doesn't yet exist + // (such as loading iframes in IE - #4833) + var documentElement = (elem ? elem.ownerDocument || elem : 0).documentElement; + + return documentElement ? documentElement.nodeName !== "HTML" : false; +}; + +var posProcess = function( selector, context ) { + var match, + tmpSet = [], + later = "", + root = context.nodeType ? [context] : context; + + // Position selectors must be done after the filter + // And so must :not(positional) so we move all PSEUDOs to the end + while ( (match = Expr.match.PSEUDO.exec( selector )) ) { + later += match[0]; + selector = selector.replace( Expr.match.PSEUDO, "" ); + } + + selector = Expr.relative[selector] ? selector + "*" : selector; + + for ( var i = 0, l = root.length; i < l; i++ ) { + Sizzle( selector, root[i], tmpSet ); + } + + return Sizzle.filter( later, tmpSet ); +}; + +// EXPOSE +jQuery.find = Sizzle; +jQuery.expr = Sizzle.selectors; +jQuery.expr[":"] = jQuery.expr.filters; +jQuery.unique = Sizzle.uniqueSort; +jQuery.text = Sizzle.getText; +jQuery.isXMLDoc = Sizzle.isXML; +jQuery.contains = Sizzle.contains; + + +})(); + + +var runtil = /Until$/, + rparentsprev = /^(?:parents|prevUntil|prevAll)/, + // Note: This RegExp should be improved, or likely pulled from Sizzle + rmultiselector = /,/, + isSimple = /^.[^:#\[\.,]*$/, + slice = Array.prototype.slice, + POS = jQuery.expr.match.POS, + // methods guaranteed to produce a unique set when starting from a unique set + guaranteedUnique = { + children: true, + contents: true, + next: true, + prev: true + }; + +jQuery.fn.extend({ + find: function( selector ) { + var self = this, + i, l; + + if ( typeof selector !== "string" ) { + return jQuery( selector ).filter(function() { + for ( i = 0, l = self.length; i < l; i++ ) { + if ( jQuery.contains( self[ i ], this ) ) { + return true; + } + } + }); + } + + var ret = this.pushStack( "", "find", selector ), + length, n, r; + + for ( i = 0, l = this.length; i < l; i++ ) { + length = ret.length; + jQuery.find( selector, this[i], ret ); + + if ( i > 0 ) { + // Make sure that the results are unique + for ( n = length; n < ret.length; n++ ) { + for ( r = 0; r < length; r++ ) { + if ( ret[r] === ret[n] ) { + ret.splice(n--, 1); + break; + } + } + } + } + } + + return ret; + }, + + has: function( target ) { + var targets = jQuery( target ); + return this.filter(function() { + for ( var i = 0, l = targets.length; i < l; i++ ) { + if ( jQuery.contains( this, targets[i] ) ) { + return true; + } + } + }); + }, + + not: function( selector ) { + return this.pushStack( winnow(this, selector, false), "not", selector); + }, + + filter: function( selector ) { + return this.pushStack( winnow(this, selector, true), "filter", selector ); + }, + + is: function( selector ) { + return !!selector && ( typeof selector === "string" ? + jQuery.filter( selector, this ).length > 0 : + this.filter( selector ).length > 0 ); + }, + + closest: function( selectors, context ) { + var ret = [], i, l, cur = this[0]; + + // Array + if ( jQuery.isArray( selectors ) ) { + var match, selector, + matches = {}, + level = 1; + + if ( cur && selectors.length ) { + for ( i = 0, l = selectors.length; i < l; i++ ) { + selector = selectors[i]; + + if ( !matches[ selector ] ) { + matches[ selector ] = POS.test( selector ) ? + jQuery( selector, context || this.context ) : + selector; + } + } + + while ( cur && cur.ownerDocument && cur !== context ) { + for ( selector in matches ) { + match = matches[ selector ]; + + if ( match.jquery ? match.index( cur ) > -1 : jQuery( cur ).is( match ) ) { + ret.push({ selector: selector, elem: cur, level: level }); + } + } + + cur = cur.parentNode; + level++; + } + } + + return ret; + } + + // String + var pos = POS.test( selectors ) || typeof selectors !== "string" ? + jQuery( selectors, context || this.context ) : + 0; + + for ( i = 0, l = this.length; i < l; i++ ) { + cur = this[i]; + + while ( cur ) { + if ( pos ? pos.index(cur) > -1 : jQuery.find.matchesSelector(cur, selectors) ) { + ret.push( cur ); + break; + + } else { + cur = cur.parentNode; + if ( !cur || !cur.ownerDocument || cur === context || cur.nodeType === 11 ) { + break; + } + } + } + } + + ret = ret.length > 1 ? jQuery.unique( ret ) : ret; + + return this.pushStack( ret, "closest", selectors ); + }, + + // Determine the position of an element within + // the matched set of elements + index: function( elem ) { + + // No argument, return index in parent + if ( !elem ) { + return ( this[0] && this[0].parentNode ) ? this.prevAll().length : -1; + } + + // index in selector + if ( typeof elem === "string" ) { + return jQuery.inArray( this[0], jQuery( elem ) ); + } + + // Locate the position of the desired element + return jQuery.inArray( + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[0] : elem, this ); + }, + + add: function( selector, context ) { + var set = typeof selector === "string" ? + jQuery( selector, context ) : + jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ), + all = jQuery.merge( this.get(), set ); + + return this.pushStack( isDisconnected( set[0] ) || isDisconnected( all[0] ) ? + all : + jQuery.unique( all ) ); + }, + + andSelf: function() { + return this.add( this.prevObject ); + } +}); + +// A painfully simple check to see if an element is disconnected +// from a document (should be improved, where feasible). +function isDisconnected( node ) { + return !node || !node.parentNode || node.parentNode.nodeType === 11; +} + +jQuery.each({ + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return jQuery.dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return jQuery.dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return jQuery.nth( elem, 2, "nextSibling" ); + }, + prev: function( elem ) { + return jQuery.nth( elem, 2, "previousSibling" ); + }, + nextAll: function( elem ) { + return jQuery.dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return jQuery.dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return jQuery.dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return jQuery.dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return jQuery.sibling( elem.parentNode.firstChild, elem ); + }, + children: function( elem ) { + return jQuery.sibling( elem.firstChild ); + }, + contents: function( elem ) { + return jQuery.nodeName( elem, "iframe" ) ? + elem.contentDocument || elem.contentWindow.document : + jQuery.makeArray( elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var ret = jQuery.map( this, fn, until ), + // The variable 'args' was introduced in + // https://github.com/jquery/jquery/commit/52a0238 + // to work around a bug in Chrome 10 (Dev) and should be removed when the bug is fixed. + // http://code.google.com/p/v8/issues/detail?id=1050 + args = slice.call(arguments); + + if ( !runtil.test( name ) ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + ret = jQuery.filter( selector, ret ); + } + + ret = this.length > 1 && !guaranteedUnique[ name ] ? jQuery.unique( ret ) : ret; + + if ( (this.length > 1 || rmultiselector.test( selector )) && rparentsprev.test( name ) ) { + ret = ret.reverse(); + } + + return this.pushStack( ret, name, args.join(",") ); + }; +}); + +jQuery.extend({ + filter: function( expr, elems, not ) { + if ( not ) { + expr = ":not(" + expr + ")"; + } + + return elems.length === 1 ? + jQuery.find.matchesSelector(elems[0], expr) ? [ elems[0] ] : [] : + jQuery.find.matches(expr, elems); + }, + + dir: function( elem, dir, until ) { + var matched = [], + cur = elem[ dir ]; + + while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { + if ( cur.nodeType === 1 ) { + matched.push( cur ); + } + cur = cur[dir]; + } + return matched; + }, + + nth: function( cur, result, dir, elem ) { + result = result || 1; + var num = 0; + + for ( ; cur; cur = cur[dir] ) { + if ( cur.nodeType === 1 && ++num === result ) { + break; + } + } + + return cur; + }, + + sibling: function( n, elem ) { + var r = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + r.push( n ); + } + } + + return r; + } +}); + +// Implement the identical functionality for filter and not +function winnow( elements, qualifier, keep ) { + + // Can't pass null or undefined to indexOf in Firefox 4 + // Set to 0 to skip string check + qualifier = qualifier || 0; + + if ( jQuery.isFunction( qualifier ) ) { + return jQuery.grep(elements, function( elem, i ) { + var retVal = !!qualifier.call( elem, i, elem ); + return retVal === keep; + }); + + } else if ( qualifier.nodeType ) { + return jQuery.grep(elements, function( elem, i ) { + return (elem === qualifier) === keep; + }); + + } else if ( typeof qualifier === "string" ) { + var filtered = jQuery.grep(elements, function( elem ) { + return elem.nodeType === 1; + }); + + if ( isSimple.test( qualifier ) ) { + return jQuery.filter(qualifier, filtered, !keep); + } else { + qualifier = jQuery.filter( qualifier, filtered ); + } + } + + return jQuery.grep(elements, function( elem, i ) { + return (jQuery.inArray( elem, qualifier ) >= 0) === keep; + }); +} + + + + +var rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g, + rleadingWhitespace = /^\s+/, + rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig, + rtagName = /<([\w:]+)/, + rtbody = /", "" ], + legend: [ 1, "
", "
" ], + thead: [ 1, "", "
" ], + tr: [ 2, "", "
" ], + td: [ 3, "", "
" ], + col: [ 2, "", "
" ], + area: [ 1, "", "" ], + _default: [ 0, "", "" ] + }; + +wrapMap.optgroup = wrapMap.option; +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + +// IE can't serialize and