Time/NL Small bug fix (#314)

Summary:
In the NL time module there's this bug where the following input

_"het is voor 5 personen"_ (english: _"it's for 5 people"_)

extracts a _time_ entity from it, instead of the expected _number_ entity.

This PR aims to fix that problem.
Pull Request resolved: https://github.com/facebook/duckling/pull/314

Reviewed By: patapizza

Differential Revision: D13742641

Pulled By: chinmay87

fbshipit-source-id: 96abb3fef4b6e5b66d0fa179751e2636a7bb4605

Duckling/Ranking/Classifiers/HE_XX.hs

@@ -934,6 +934,13 @@ classifiers
                    koData =
                      ClassData{prior = -infinity, unseen = -1.6094379124341003,
                                likelihoods = HashMap.fromList [], n = 0}}),
+       ("half",
+        Classifier{okData =
+                     ClassData{prior = -infinity, unseen = -0.6931471805599453,
+                               likelihoods = HashMap.fromList [], n = 0},
+                   koData =
+                     ClassData{prior = 0.0, unseen = -1.0986122886681098,
+                               likelihoods = HashMap.fromList [("", 0.0)], n = 1}}),
        ("ordinal 5",
         Classifier{okData =
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,

Duckling/Ranking/Classifiers/NL_BE.hs

@@ -1401,20 +1401,6 @@ classifiers
                    koData =
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,
                                likelihoods = HashMap.fromList [], n = 0}}),
-       ("by the end of <time>",
-        Classifier{okData =
-                     ClassData{prior = -infinity, unseen = -1.791759469228055,
-                               likelihoods = HashMap.fromList [], n = 0},
-                   koData =
-                     ClassData{prior = 0.0, unseen = -2.890371757896165,
-                               likelihoods =
-                                 HashMap.fromList
-                                   [("half to|till|before <integer> (hour-of-day)",
-                                     -1.7346010553881064),
-                                    ("time-of-day (latent)", -1.4469189829363254),
-                                    ("hh:mm", -2.1400661634962708), ("hour", -1.4469189829363254),
-                                    ("minute", -1.4469189829363254)],
-                               n = 6}}),
        ("in <duration>",
         Classifier{okData =
                      ClassData{prior = -4.652001563489282e-2,

Duckling/Ranking/Classifiers/NL_NL.hs

@@ -1394,20 +1394,6 @@ classifiers
                    koData =
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,
                                likelihoods = HashMap.fromList [], n = 0}}),
-       ("by the end of <time>",
-        Classifier{okData =
-                     ClassData{prior = -infinity, unseen = -1.791759469228055,
-                               likelihoods = HashMap.fromList [], n = 0},
-                   koData =
-                     ClassData{prior = 0.0, unseen = -2.890371757896165,
-                               likelihoods =
-                                 HashMap.fromList
-                                   [("half to|till|before <integer> (hour-of-day)",
-                                     -1.7346010553881064),
-                                    ("time-of-day (latent)", -1.4469189829363254),
-                                    ("hh:mm", -2.1400661634962708), ("hour", -1.4469189829363254),
-                                    ("minute", -1.4469189829363254)],
-                               n = 6}}),
        ("in <duration>",
         Classifier{okData =
                      ClassData{prior = -4.652001563489282e-2,

Duckling/Ranking/Classifiers/NL_XX.hs

@@ -1401,20 +1401,6 @@ classifiers
                    koData =
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,
                                likelihoods = HashMap.fromList [], n = 0}}),
-       ("by the end of <time>",
-        Classifier{okData =
-                     ClassData{prior = -infinity, unseen = -1.791759469228055,
-                               likelihoods = HashMap.fromList [], n = 0},
-                   koData =
-                     ClassData{prior = 0.0, unseen = -2.890371757896165,
-                               likelihoods =
-                                 HashMap.fromList
-                                   [("half to|till|before <integer> (hour-of-day)",
-                                     -1.7346010553881064),
-                                    ("time-of-day (latent)", -1.4469189829363254),
-                                    ("hh:mm", -2.1400661634962708), ("hour", -1.4469189829363254),
-                                    ("minute", -1.4469189829363254)],
-                               n = 6}}),
        ("in <duration>",
         Classifier{okData =
                      ClassData{prior = -4.652001563489282e-2,

Duckling/Time/NL/Corpus.hs

@@ -11,6 +11,7 @@
 module Duckling.Time.NL.Corpus
   ( corpus
   , defaultCorpus
+  , latentCorpus
   , negativeCorpus
   ) where
 
@@ -41,6 +42,15 @@ defaultCorpus = (context, testOptions, allExamples ++ custom)
                  ]
       ]
 
+latentCorpus :: Corpus
+latentCorpus = (context, testOptions {withLatent = True}, xs)
+  where
+    xs = concat
+      [ examples (datetimeOpenInterval Before (2013, 2, 12, 8, 0, 0) Hour)
+                 [ "voor 8"
+                 ]
+      ]
+
 negativeCorpus :: NegativeCorpus
 negativeCorpus = (context, testOptions, examples)
   where

Duckling/Time/NL/Rules.hs

@@ -711,7 +711,7 @@ ruleByTheEndOfTime :: Rule
 ruleByTheEndOfTime = Rule
   { name = "by the end of <time>"
   , pattern =
-    [ regex "tot (het)? einde (van)?|voor"
+    [ regex "tot( het)? einde( van)?"
     , dimension Time
     ]
   , prod = \tokens -> case tokens of

tests/Duckling/Time/NL/Tests.hs

@@ -24,6 +24,7 @@ import qualified Duckling.Time.NL.BE.Corpus as BE
 tests :: TestTree
 tests = testGroup "NL Tests"
   [ makeCorpusTest [This Time] corpus
+  , makeCorpusTest [This Time] latentCorpus
   , makeNegativeCorpusTest [This Time] negativeCorpus
   , localeTests
   ]