Add Numeral dimension for new language KM. (#234)

Summary:
Hello,
I am new to Haskell, but I would like to add Khmer language (KM) to Duckling.
I have tried to extended Duckling by adding Numeral dimension for new language KM.
Please have a look at it and see what we can improve.

Thanks!
Pull Request resolved: https://github.com/facebook/duckling/pull/234

Reviewed By: blandinw

Differential Revision: D9032639

Pulled By: chinmay87

fbshipit-source-id: 7db19edf732fe6500629cc89e18e0655d7bbc48b

Duckling/Dimensions.hs

@@ -43,6 +43,7 @@ import qualified Duckling.Dimensions.ID as IDDimensions
 import qualified Duckling.Dimensions.IT as ITDimensions
 import qualified Duckling.Dimensions.JA as JADimensions
 import qualified Duckling.Dimensions.KA as KADimensions
+import qualified Duckling.Dimensions.KM as KMDimensions
 import qualified Duckling.Dimensions.KO as KODimensions
 import qualified Duckling.Dimensions.MY as MYDimensions
 import qualified Duckling.Dimensions.NB as NBDimensions
@@ -109,6 +110,7 @@ langDimensions ID = IDDimensions.allDimensions
 langDimensions IT = ITDimensions.allDimensions
 langDimensions JA = JADimensions.allDimensions
 langDimensions KA = KADimensions.allDimensions
+langDimensions KM = KMDimensions.allDimensions
 langDimensions KO = KODimensions.allDimensions
 langDimensions MY = MYDimensions.allDimensions
 langDimensions NB = NBDimensions.allDimensions

Duckling/Dimensions/KM.hs

@@ -0,0 +1,17 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+module Duckling.Dimensions.KM
+  ( allDimensions
+  ) where
+
+import Duckling.Dimensions.Types
+
+allDimensions :: [Some Dimension]
+allDimensions =
+  [ This Numeral ]

Duckling/Locale.hs

@@ -55,6 +55,7 @@ data Lang
   | IT
   | JA
   | KA
+  | KM
   | KO
   | MY
   | NB

Duckling/Numeral/KM/Corpus.hs

@@ -0,0 +1,70 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Duckling.Numeral.KM.Corpus
+  ( corpus ) where
+
+import Data.String
+import Prelude
+
+import Duckling.Locale
+import Duckling.Numeral.Types
+import Duckling.Resolve
+import Duckling.Testing.Types
+
+context :: Context
+context = testContext {locale = makeLocale KM Nothing}
+
+corpus :: Corpus
+corpus = (context, testOptions, allExamples)
+
+allExamples :: [Example]
+allExamples = concat
+  [ examples (NumeralValue 0)
+             [ "0"
+             , "០"
+             ]
+  , examples (NumeralValue 1)
+             [ "1"
+             , "១"
+             ]
+  , examples (NumeralValue 2)
+             [ "2"
+             , "២"
+             ]
+  , examples (NumeralValue 3)
+             [ "3"
+             , "៣"
+             ]
+  , examples (NumeralValue 4)
+             [ "4"
+             , "៤"
+             ]
+  , examples (NumeralValue 5)
+             [ "5"
+             , "៥"
+             ]
+  , examples (NumeralValue 6)
+             [ "6"
+             , "៦"
+             ]
+  , examples (NumeralValue 7)
+             [ "7"
+             , "៧"
+             ]
+  , examples (NumeralValue 8)
+             [ "8"
+             , "៨"
+             ]
+  , examples (NumeralValue 9)
+             [ "9"
+             , "៩"
+             ]
+  ]

Duckling/Numeral/KM/Rules.hs

@@ -0,0 +1,60 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE NoRebindableSyntax #-}
+
+module Duckling.Numeral.KM.Rules
+  ( rules ) where
+
+import Data.HashMap.Strict (HashMap)
+import Data.Maybe
+import Data.String
+import Data.Text (Text)
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.Text as Text
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.Helpers
+import Duckling.Numeral.Types (NumeralData (..))
+import Duckling.Regex.Types
+import Duckling.Types
+import qualified Duckling.Numeral.Types as TNumeral
+
+ruleNumeralMap :: HashMap Text Integer
+ruleNumeralMap = HashMap.fromList
+  [ ( "០", 0 )
+  , ( "១", 1 )
+  , ( "២", 2 )
+  , ( "៣", 3 )
+  , ( "៤", 4 )
+  , ( "៥", 5 )
+  , ( "៦", 6 )
+  , ( "៧", 7 )
+  , ( "៨", 8 )
+  , ( "៩", 9 )
+  ]
+
+ruleNumeral :: Rule
+ruleNumeral = Rule
+  { name = "number (0..9)"
+  , pattern =
+    [ regex "(០|១|២|៣|៤|៥|៦|៧|៨|៩)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup match ruleNumeralMap >>= integer
+      _ -> Nothing
+  }
+
+rules :: [Rule]
+rules =
+  [ ruleNumeral
+  ]

Duckling/Ranking/Classifiers.hs

@@ -37,6 +37,7 @@ import qualified Duckling.Ranking.Classifiers.ID_XX as ID_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.IT_XX as IT_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.JA_XX as JA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KA_XX as KA_XXClassifiers
+import qualified Duckling.Ranking.Classifiers.KM_XX as KM_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KO_XX as KO_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.MY_XX as MY_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.NB_XX as NB_XXClassifiers
@@ -77,6 +78,7 @@ classifiers (Locale ID _) = ID_XXClassifiers.classifiers
 classifiers (Locale IT _) = IT_XXClassifiers.classifiers
 classifiers (Locale JA _) = JA_XXClassifiers.classifiers
 classifiers (Locale KA _) = KA_XXClassifiers.classifiers
+classifiers (Locale KM _) = KM_XXClassifiers.classifiers
 classifiers (Locale KO _) = KO_XXClassifiers.classifiers
 classifiers (Locale MY _) = MY_XXClassifiers.classifiers
 classifiers (Locale NB _) = NB_XXClassifiers.classifiers

Duckling/Ranking/Classifiers/KM_XX.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+-----------------------------------------------------------------
+-- Auto-generated by regenClassifiers
+--
+-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+--  @generated
+-----------------------------------------------------------------
+{-# LANGUAGE OverloadedStrings #-}
+module Duckling.Ranking.Classifiers.KM_XX (classifiers) where
+import Data.String
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import Duckling.Ranking.Types
+
+classifiers :: Classifiers
+classifiers = HashMap.fromList []

Duckling/Rules.hs

@@ -44,6 +44,7 @@ import qualified Duckling.Rules.ID as IDRules
 import qualified Duckling.Rules.IT as ITRules
 import qualified Duckling.Rules.JA as JARules
 import qualified Duckling.Rules.KA as KARules
+import qualified Duckling.Rules.KM as KMRules
 import qualified Duckling.Rules.KO as KORules
 import qualified Duckling.Rules.MY as MYRules
 import qualified Duckling.Rules.NB as NBRules
@@ -102,6 +103,7 @@ defaultRules ID = IDRules.defaultRules
 defaultRules IT = ITRules.defaultRules
 defaultRules JA = JARules.defaultRules
 defaultRules KA = KARules.defaultRules
+defaultRules KM = KMRules.defaultRules
 defaultRules KO = KORules.defaultRules
 defaultRules MY = MYRules.defaultRules
 defaultRules NB = NBRules.defaultRules
@@ -140,6 +142,7 @@ localeRules ID = IDRules.localeRules
 localeRules IT = ITRules.localeRules
 localeRules JA = JARules.localeRules
 localeRules KA = KARules.localeRules
+localeRules KM = KMRules.localeRules
 localeRules KO = KORules.localeRules
 localeRules MY = MYRules.localeRules
 localeRules NB = NBRules.localeRules
@@ -178,6 +181,7 @@ langRules ID = IDRules.langRules
 langRules IT = ITRules.langRules
 langRules JA = JARules.langRules
 langRules KA = KARules.langRules
+langRules KM = KMRules.langRules
 langRules KO = KORules.langRules
 langRules MY = MYRules.langRules
 langRules NB = NBRules.langRules

Duckling/Rules/KM.hs

@@ -0,0 +1,45 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+
+
+module Duckling.Rules.KM
+  ( defaultRules
+  , langRules
+  , localeRules
+  ) where
+
+import Duckling.Dimensions.Types
+import Duckling.Locale
+import Duckling.Types
+import qualified Duckling.Numeral.KM.Rules as Numeral
+
+defaultRules :: Some Dimension -> [Rule]
+defaultRules = langRules
+
+localeRules :: Region -> Some Dimension -> [Rule]
+localeRules region (This (CustomDimension dim)) = dimLocaleRules region dim
+localeRules _ _ = []
+
+langRules :: Some Dimension -> [Rule]
+langRules (This AmountOfMoney) = []
+langRules (This Distance) = []
+langRules (This Duration) = []
+langRules (This Email) = []
+langRules (This Numeral) = Numeral.rules
+langRules (This Ordinal) = []
+langRules (This PhoneNumber) = []
+langRules (This Quantity) = []
+langRules (This RegexMatch) = []
+langRules (This Temperature) = []
+langRules (This Time) = []
+langRules (This TimeGrain) = []
+langRules (This Url) = []
+langRules (This Volume) = []
+langRules (This (CustomDimension dim)) = dimLangRules KM dim

duckling.cabal

@@ -62,6 +62,7 @@ library
                      , Duckling.Rules.IT
                      , Duckling.Rules.JA
                      , Duckling.Rules.KA
+                     , Duckling.Rules.KM
                      , Duckling.Rules.KO
                      , Duckling.Rules.MY
                      , Duckling.Rules.NB
@@ -108,6 +109,7 @@ library
                      , Duckling.Ranking.Classifiers.IT_XX
                      , Duckling.Ranking.Classifiers.JA_XX
                      , Duckling.Ranking.Classifiers.KA_XX
+                     , Duckling.Ranking.Classifiers.KM_XX
                      , Duckling.Ranking.Classifiers.KO_XX
                      , Duckling.Ranking.Classifiers.MY_XX
                      , Duckling.Ranking.Classifiers.NB_XX
@@ -156,6 +158,7 @@ library
                      , Duckling.Dimensions.IT
                      , Duckling.Dimensions.JA
                      , Duckling.Dimensions.KA
+                     , Duckling.Dimensions.KM
                      , Duckling.Dimensions.KO
                      , Duckling.Dimensions.MY
                      , Duckling.Dimensions.NB
@@ -369,6 +372,8 @@ library
                      , Duckling.Numeral.JA.Rules
                      , Duckling.Numeral.KA.Corpus
                      , Duckling.Numeral.KA.Rules
+                     , Duckling.Numeral.KM.Corpus
+                     , Duckling.Numeral.KM.Rules
                      , Duckling.Numeral.KO.Corpus
                      , Duckling.Numeral.KO.Rules
                      , Duckling.Numeral.MY.Corpus
@@ -813,6 +818,7 @@ test-suite duckling-test
                      , Duckling.Numeral.IT.Tests
                      , Duckling.Numeral.JA.Tests
                      , Duckling.Numeral.KA.Tests
+                     , Duckling.Numeral.KM.Tests
                      , Duckling.Numeral.KO.Tests
                      , Duckling.Numeral.MY.Tests
                      , Duckling.Numeral.NB.Tests

exe/Duckling/Ranking/Generate.hs

@@ -190,6 +190,7 @@ getCorpusForLang ID = (testContext, testOptions, [])
 getCorpusForLang IT = ITTime.corpus
 getCorpusForLang JA = (testContext, testOptions, [])
 getCorpusForLang KA = (testContext, testOptions, [])
+getCorpusForLang KM = (testContext, testOptions, [])
 getCorpusForLang KO = KOTime.corpus
 getCorpusForLang MY = (testContext, testOptions, [])
 getCorpusForLang NB = NBTime.corpus

tests/Duckling/Numeral/KM/Tests.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+module Duckling.Numeral.KM.Tests
+  ( tests ) where
+
+import Data.String
+import Prelude
+import Test.Tasty
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.KM.Corpus
+import Duckling.Testing.Asserts
+
+tests :: TestTree
+tests = testGroup "KM Tests"
+  [ makeCorpusTest [This Numeral] corpus
+  ]

tests/Duckling/Numeral/Tests.hs

@@ -33,6 +33,7 @@ import qualified Duckling.Numeral.ID.Tests as ID
 import qualified Duckling.Numeral.IT.Tests as IT
 import qualified Duckling.Numeral.JA.Tests as JA
 import qualified Duckling.Numeral.KA.Tests as KA
+import qualified Duckling.Numeral.KM.Tests as KM
 import qualified Duckling.Numeral.KO.Tests as KO
 import qualified Duckling.Numeral.MY.Tests as MY
 import qualified Duckling.Numeral.NB.Tests as NB
@@ -72,6 +73,7 @@ tests = testGroup "Numeral Tests"
   , IT.tests
   , JA.tests
   , KA.tests
+  , KM.tests
   , KO.tests
   , MY.tests
   , NB.tests