Added support Lao numerals

Summary: Duckling didn't support Lao numerals, this diff adds it.

Reviewed By: patapizza

Differential Revision: D9323242

fbshipit-source-id: 5dad0d4dfb4843281a327947690e664c91ab8f1a

Duckling/Dimensions.hs

@@ -45,6 +45,7 @@ import qualified Duckling.Dimensions.JA as JADimensions
 import qualified Duckling.Dimensions.KA as KADimensions
 import qualified Duckling.Dimensions.KM as KMDimensions
 import qualified Duckling.Dimensions.KO as KODimensions
+import qualified Duckling.Dimensions.LO as LODimensions
 import qualified Duckling.Dimensions.MY as MYDimensions
 import qualified Duckling.Dimensions.NB as NBDimensions
 import qualified Duckling.Dimensions.NE as NEDimensions
@@ -112,6 +113,7 @@ langDimensions JA = JADimensions.allDimensions
 langDimensions KA = KADimensions.allDimensions
 langDimensions KM = KMDimensions.allDimensions
 langDimensions KO = KODimensions.allDimensions
+langDimensions LO = LODimensions.allDimensions
 langDimensions MY = MYDimensions.allDimensions
 langDimensions NB = NBDimensions.allDimensions
 langDimensions NE = NEDimensions.allDimensions

Duckling/Dimensions/LO.hs

@@ -0,0 +1,18 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+module Duckling.Dimensions.LO
+  ( allDimensions
+  ) where
+
+import Duckling.Dimensions.Types
+
+allDimensions :: [Some Dimension]
+allDimensions =
+  [ This Numeral
+  ]

Duckling/Locale.hs

@@ -57,6 +57,7 @@ data Lang
   | KA
   | KM
   | KO
+  | LO
   | MY
   | NB
   | NE

Duckling/Numeral/LO/Corpus.hs

@@ -0,0 +1,90 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Duckling.Numeral.LO.Corpus
+  ( corpus ) where
+
+import Data.String
+import Prelude
+
+import Duckling.Locale
+import Duckling.Numeral.Types
+import Duckling.Resolve
+import Duckling.Testing.Types
+
+corpus :: Corpus
+corpus = (testContext {locale = makeLocale LO Nothing}, testOptions, allExamples)
+
+allExamples :: [Example]
+allExamples = concat
+  [ examples (NumeralValue 0)
+             [ "ສູນ"
+             ]
+  , examples (NumeralValue 1)
+             [ "ໜຶ່ງ"
+             ]
+  , examples (NumeralValue 2)
+             [ "ສອງ"
+             ]
+  , examples (NumeralValue 3)
+             [ "ສາມ"
+             ]
+  , examples (NumeralValue 4)
+             [ "ສີ່"
+             ]
+  , examples (NumeralValue 5)
+             [ "ຫ້າ"
+             ]
+  , examples (NumeralValue 6)
+             [ "ຫົກ"
+             ]
+  , examples (NumeralValue 7)
+             [ "ເຈັດ"
+             ]
+  , examples (NumeralValue 8)
+             [ "ແປດ"
+             ]
+  , examples (NumeralValue 9)
+             [ "ເກົ້າ"
+             ]
+  , examples (NumeralValue 11)
+             [ "ສິບເອັດ"
+             ]
+  , examples (NumeralValue 15)
+             [ "ສິບຫ້າ"
+             ]
+  , examples (NumeralValue 17)
+             [ "ສິບເຈັດ"
+             ]
+  , examples (NumeralValue 22)
+             [ "ຊາວສອງ"
+             ]
+  , examples (NumeralValue 24)
+             [ "ຊາວສີ່"
+             ]
+  , examples (NumeralValue 26)
+             [ "ຊາວຫົກ"
+             ]
+  , examples (NumeralValue 28)
+             [ "ຊາວແປດ"
+             ]
+  , examples (NumeralValue 34)
+             [ "ສາມສິບສີ່"
+             ]
+  , examples (NumeralValue 10)
+             [ "ສິບ"
+             ]
+  , examples (NumeralValue 20)
+             [ "ຊາວ"
+             ]
+  , examples (NumeralValue 50)
+             [ "ຫ້າສິບ"
+             ]
+  ]

Duckling/Numeral/LO/Rules.hs

@@ -0,0 +1,154 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE NoRebindableSyntax #-}
+
+module Duckling.Numeral.LO.Rules
+  ( rules
+  ) where
+
+import Data.HashMap.Strict (HashMap)
+import Data.Maybe
+import Data.String
+import Data.Text (Text)
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.Text as Text
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.Helpers
+import Duckling.Numeral.Types (NumeralData (..))
+import Duckling.Regex.Types
+import Duckling.Types
+import qualified Duckling.Numeral.Types as TNumeral
+
+ruleNumeralMap :: HashMap Text Integer
+ruleNumeralMap = HashMap.fromList
+  [ ( "ສູນ", 0 )
+  , ( "ໜຶ່ງ", 1 )
+  , ( "ສອງ", 2 )
+  , ( "ສາມ", 3 )
+  , ( "ສີ່", 4 )
+  , ( "ຫ້າ", 5)
+  , ( "ຫົກ", 6)
+  , ( "ເຈັດ", 7)
+  , ( "ແປດ", 8)
+  , ( "ເກົ້າ", 9)
+  , ( "ສິບ", 10)
+  ]
+
+ruleNumeral :: Rule
+ruleNumeral = Rule
+  { name = "number (0..10)"
+  , pattern =
+    [ regex "(ສູນ|ໜຶ່ງ|ສອງ|ສາມ|ສີ່|ຫ້າ|ຫົກ|ເຈັດ|ແປດ|ເກົ້າ|ສິບ)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) ruleNumeralMap >>= integer
+      _ -> Nothing
+  }
+
+elevenToNineteenMap :: HashMap Text Integer
+elevenToNineteenMap = HashMap.fromList
+  [ ( "ສິບເອັດ", 11 )
+  , ( "ສິບສອງ", 12 )
+  , ( "ສິບສາມ", 13 )
+  , ( "ສິບສີ່", 14 )
+  , ( "ສິບຫ້າ", 15 )
+  , ( "ສິບຫົກ", 16 )
+  , ( "ສິບເຈັດ", 17 )
+  , ( "ສິບແປດ", 18 )
+  , ( "ສິບເກົ້າ", 19 )
+  ]
+
+ruleElevenToNineteen :: Rule
+ruleElevenToNineteen = Rule
+  { name = "number (11..19)"
+  , pattern =
+    [ regex "(ສິບເອັດ|ສິບສອງ|ສິບສາມ|ສິບສີ່|ສິບຫ້າ|ສິບຫົກ|ສິບເຈັດ|ສິບແປດ|ສິບເກົ້າ)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) elevenToNineteenMap >>= integer
+      _ -> Nothing
+  }
+
+twentyoneToTwentynineMap :: HashMap Text Integer
+twentyoneToTwentynineMap = HashMap.fromList
+  [ ( "ຊາວເອັດ", 21 )
+  , ( "ຊາວສອງ", 22 )
+  , ( "ຊາວສາມ", 23 )
+  , ( "ຊາວສີ່", 24 )
+  , ( "ຊາວຫ້າ", 25 )
+  , ( "ຊາວຫົກ", 26 )
+  , ( "ຊາວເຈັດ", 27 )
+  , ( "ຊາວແປດ", 28 )
+  , ( "ຊາວເກົ້າ", 29 )
+  ]
+
+ruleTwentyoneToTwentynine :: Rule
+ruleTwentyoneToTwentynine = Rule
+  { name = "number (21..29)"
+  , pattern =
+    [ regex "(ຊາວເອັດ|ຊາວສອງ|ຊາວສາມ|ຊາວສີ່|ຊາວຫ້າ|ຊາວຫົກ|ຊາວເຈັດ|ຊາວແປດ|ຊາວເກົ້າ)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) twentyoneToTwentynineMap >>= integer
+      _ -> Nothing
+  }
+
+tensMap :: HashMap Text Integer
+tensMap = HashMap.fromList
+  [ ( "ຊາວ", 20 )
+  , ( "ສາມສິບ", 30 )
+  , ( "ສິບສີ່", 40 )
+  , ( "ຫ້າສິບ", 50 )
+  , ( "ຫົກສິບ", 60 )
+  , ( "ເຈັດສິບ", 70 )
+  , ( "ແປດສິບ", 80 )
+  , ( "ເກົ້າສິບ", 90 )
+  ]
+
+ruleTens :: Rule
+ruleTens = Rule
+  { name = "integer (20,30..90)"
+  , pattern =
+    [ regex "(ຊາວ|ສາມສິບ|ສິບສີ່|ຫ້າສິບ|ຫົກສິບ|ເຈັດສິບ|ແປດສິບ|ເກົ້າສິບ)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) tensMap >>= integer
+      _ -> Nothing
+  }
+
+ruleCompositeTens :: Rule
+ruleCompositeTens = Rule
+  { name = "integer ([3-9][1-9])"
+  , pattern =
+    [ regex "(ສາມສິບ|ສິບສີ່|ຫ້າສິບ|ຫົກສິບ|ເຈັດສິບ|ແປດສິບ|ເກົ້າສິບ)(ໜຶ່ງ|ສອງ|ສາມ|ສີ່|ຫ້າ|ຫົກ|ເຈັດ|ແປດ|ເກົ້າ)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (m1:m2:_)):_) -> do
+        v1 <- HashMap.lookup (Text.toLower m1) tensMap
+        v2 <- HashMap.lookup (Text.toLower m2) ruleNumeralMap
+        integer $ v1 + v2
+      _ -> Nothing
+  }
+
+rules :: [Rule]
+rules =
+  [ ruleNumeral
+  , ruleElevenToNineteen
+  , ruleTwentyoneToTwentynine
+  , ruleTens
+  , ruleCompositeTens
+  ]

Duckling/Ranking/Classifiers.hs

@@ -39,6 +39,7 @@ import qualified Duckling.Ranking.Classifiers.JA_XX as JA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KA_XX as KA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KM_XX as KM_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KO_XX as KO_XXClassifiers
+import qualified Duckling.Ranking.Classifiers.LO_XX as LO_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.MY_XX as MY_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.NB_XX as NB_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.NE_XX as NE_XXClassifiers
@@ -80,6 +81,7 @@ classifiers (Locale JA _) = JA_XXClassifiers.classifiers
 classifiers (Locale KA _) = KA_XXClassifiers.classifiers
 classifiers (Locale KM _) = KM_XXClassifiers.classifiers
 classifiers (Locale KO _) = KO_XXClassifiers.classifiers
+classifiers (Locale LO _) = LO_XXClassifiers.classifiers
 classifiers (Locale MY _) = MY_XXClassifiers.classifiers
 classifiers (Locale NB _) = NB_XXClassifiers.classifiers
 classifiers (Locale NE _) = NE_XXClassifiers.classifiers

Duckling/Ranking/Classifiers/LO_XX.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+-----------------------------------------------------------------
+-- Auto-generated by regenClassifiers
+--
+-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+--  @generated
+-----------------------------------------------------------------
+{-# LANGUAGE OverloadedStrings #-}
+module Duckling.Ranking.Classifiers.LO_XX (classifiers) where
+import Data.String
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import Duckling.Ranking.Types
+
+classifiers :: Classifiers
+classifiers = HashMap.fromList []

Duckling/Rules.hs

@@ -46,6 +46,7 @@ import qualified Duckling.Rules.JA as JARules
 import qualified Duckling.Rules.KA as KARules
 import qualified Duckling.Rules.KM as KMRules
 import qualified Duckling.Rules.KO as KORules
+import qualified Duckling.Rules.LO as LORules
 import qualified Duckling.Rules.MY as MYRules
 import qualified Duckling.Rules.NB as NBRules
 import qualified Duckling.Rules.NE as NERules
@@ -105,6 +106,7 @@ defaultRules JA = JARules.defaultRules
 defaultRules KA = KARules.defaultRules
 defaultRules KM = KMRules.defaultRules
 defaultRules KO = KORules.defaultRules
+defaultRules LO = LORules.defaultRules
 defaultRules MY = MYRules.defaultRules
 defaultRules NB = NBRules.defaultRules
 defaultRules NE = NERules.defaultRules
@@ -144,6 +146,7 @@ localeRules JA = JARules.localeRules
 localeRules KA = KARules.localeRules
 localeRules KM = KMRules.localeRules
 localeRules KO = KORules.localeRules
+localeRules LO = LORules.localeRules
 localeRules MY = MYRules.localeRules
 localeRules NB = NBRules.localeRules
 localeRules NE = NERules.localeRules
@@ -183,6 +186,7 @@ langRules JA = JARules.langRules
 langRules KA = KARules.langRules
 langRules KM = KMRules.langRules
 langRules KO = KORules.langRules
+langRules LO = LORules.langRules
 langRules MY = MYRules.langRules
 langRules NB = NBRules.langRules
 langRules NE = NERules.langRules

Duckling/Rules/LO.hs

@@ -0,0 +1,45 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+
+
+module Duckling.Rules.LO
+  ( defaultRules
+  , langRules
+  , localeRules
+  ) where
+
+import Duckling.Dimensions.Types
+import Duckling.Locale
+import Duckling.Types
+import qualified Duckling.Numeral.LO.Rules as Numeral
+
+defaultRules :: Some Dimension -> [Rule]
+defaultRules = langRules
+
+localeRules :: Region -> Some Dimension -> [Rule]
+localeRules region (This (CustomDimension dim)) = dimLocaleRules region dim
+localeRules _ _ = []
+
+langRules :: Some Dimension -> [Rule]
+langRules (This AmountOfMoney) = []
+langRules (This Distance) = []
+langRules (This Duration) = []
+langRules (This Email) = []
+langRules (This Numeral) = Numeral.rules
+langRules (This Ordinal) = []
+langRules (This PhoneNumber) = []
+langRules (This Quantity) = []
+langRules (This RegexMatch) = []
+langRules (This Temperature) = []
+langRules (This Time) = []
+langRules (This TimeGrain) = []
+langRules (This Url) = []
+langRules (This Volume) = []
+langRules (This (CustomDimension dim)) = dimLangRules LO dim

duckling.cabal

@@ -64,6 +64,7 @@ library
                      , Duckling.Rules.KA
                      , Duckling.Rules.KM
                      , Duckling.Rules.KO
+                     , Duckling.Rules.LO
                      , Duckling.Rules.MY
                      , Duckling.Rules.NB
                      , Duckling.Rules.NE
@@ -111,6 +112,7 @@ library
                      , Duckling.Ranking.Classifiers.KA_XX
                      , Duckling.Ranking.Classifiers.KM_XX
                      , Duckling.Ranking.Classifiers.KO_XX
+                     , Duckling.Ranking.Classifiers.LO_XX
                      , Duckling.Ranking.Classifiers.MY_XX
                      , Duckling.Ranking.Classifiers.NB_XX
                      , Duckling.Ranking.Classifiers.NE_XX
@@ -160,6 +162,7 @@ library
                      , Duckling.Dimensions.KA
                      , Duckling.Dimensions.KM
                      , Duckling.Dimensions.KO
+                     , Duckling.Dimensions.LO
                      , Duckling.Dimensions.MY
                      , Duckling.Dimensions.NB
                      , Duckling.Dimensions.NE
@@ -376,6 +379,8 @@ library
                      , Duckling.Numeral.KM.Rules
                      , Duckling.Numeral.KO.Corpus
                      , Duckling.Numeral.KO.Rules
+                     , Duckling.Numeral.LO.Corpus
+                     , Duckling.Numeral.LO.Rules
                      , Duckling.Numeral.MY.Corpus
                      , Duckling.Numeral.MY.Rules
                      , Duckling.Numeral.NB.Corpus
@@ -820,6 +825,7 @@ test-suite duckling-test
                      , Duckling.Numeral.KA.Tests
                      , Duckling.Numeral.KM.Tests
                      , Duckling.Numeral.KO.Tests
+                     , Duckling.Numeral.LO.Tests
                      , Duckling.Numeral.MY.Tests
                      , Duckling.Numeral.NB.Tests
                      , Duckling.Numeral.NE.Tests

exe/Duckling/Ranking/Generate.hs

@@ -192,6 +192,7 @@ getCorpusForLang JA = (testContext, testOptions, [])
 getCorpusForLang KA = (testContext, testOptions, [])
 getCorpusForLang KM = (testContext, testOptions, [])
 getCorpusForLang KO = KOTime.corpus
+getCorpusForLang LO = (testContext, testOptions, [])
 getCorpusForLang MY = (testContext, testOptions, [])
 getCorpusForLang NB = NBTime.corpus
 getCorpusForLang NE = (testContext, testOptions, [])

tests/Duckling/Numeral/LO/Tests.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+module Duckling.Numeral.LO.Tests
+  ( tests ) where
+
+import Data.String
+import Prelude
+import Test.Tasty
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.LO.Corpus
+import Duckling.Testing.Asserts
+
+tests :: TestTree
+tests = testGroup "LO Tests"
+  [ makeCorpusTest [This Numeral] corpus
+  ]

tests/Duckling/Numeral/Tests.hs

@@ -35,6 +35,7 @@ import qualified Duckling.Numeral.JA.Tests as JA
 import qualified Duckling.Numeral.KA.Tests as KA
 import qualified Duckling.Numeral.KM.Tests as KM
 import qualified Duckling.Numeral.KO.Tests as KO
+import qualified Duckling.Numeral.LO.Tests as LO
 import qualified Duckling.Numeral.MY.Tests as MY
 import qualified Duckling.Numeral.NB.Tests as NB
 import qualified Duckling.Numeral.NE.Tests as NE
@@ -75,6 +76,7 @@ tests = testGroup "Numeral Tests"
   , KA.tests
   , KM.tests
   , KO.tests
+  , LO.tests
   , MY.tests
   , NB.tests
   , NE.tests