rsennrich/subword-nmt
1
1
Fork 0
mirror of https://github.com/rsennrich/subword-nmt.git synced 2024-11-27 02:53:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

break condition for toy example

Browse Source
This commit is contained in:
Rico Sennrich 2016-03-03 16:39:34 +00:00
parent d0c78f57c8
commit 5d2d3758ad
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bpe_toy.py
View File

@ -13,7 +13,9 @@ Rico Sennrich, Barry Haddow and Alexandra Birch (2015). Neural Machine Translati
"""
import re, collections
import re
import sys
import collections
def get_stats(vocab):
pairs = collections.defaultdict(int)
@ -38,5 +40,8 @@ num_merges = 15
for i in range(num_merges):
pairs = get_stats(vocab)
best = max(pairs, key=pairs.get)
if pairs[best] < 2:
sys.stderr.write('no pair has frequency > 1. Stopping\n')
break
vocab = merge_vocab(best, vocab)
print(best)