add Kannada support (for Numeral)

Summary: Pull Request resolved: https://github.com/facebook/duckling/pull/322

Reviewed By: patapizza

Differential Revision: D13888983

Pulled By: haoxuany

fbshipit-source-id: b7a3b5c72dae0ccc637cf6faa14670ebf868bb50

Duckling/Dimensions.hs

@@ -45,6 +45,7 @@ import qualified Duckling.Dimensions.IT as ITDimensions
 import qualified Duckling.Dimensions.JA as JADimensions
 import qualified Duckling.Dimensions.KA as KADimensions
 import qualified Duckling.Dimensions.KM as KMDimensions
+import qualified Duckling.Dimensions.KN as KNDimensions
 import qualified Duckling.Dimensions.KO as KODimensions
 import qualified Duckling.Dimensions.LO as LODimensions
 import qualified Duckling.Dimensions.ML as MLDimensions
@@ -119,6 +120,7 @@ langDimensions IT = ITDimensions.allDimensions
 langDimensions JA = JADimensions.allDimensions
 langDimensions KA = KADimensions.allDimensions
 langDimensions KM = KMDimensions.allDimensions
+langDimensions KN = KNDimensions.allDimensions
 langDimensions KO = KODimensions.allDimensions
 langDimensions LO = LODimensions.allDimensions
 langDimensions ML = MLDimensions.allDimensions

Duckling/Dimensions/KN.hs

@@ -0,0 +1,18 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+module Duckling.Dimensions.KN
+  ( allDimensions
+  ) where
+
+import Duckling.Dimensions.Types
+
+allDimensions :: [Some Dimension]
+allDimensions =
+  [ This Numeral
+  ]

Duckling/Locale.hs

@@ -56,6 +56,7 @@ data Lang
   | IT
   | JA
   | KA
+  | KN
   | KM
   | KO
   | LO

Duckling/Numeral/KN/Corpus.hs

@@ -0,0 +1,68 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Duckling.Numeral.KN.Corpus
+  ( corpus
+  ) where
+
+import Data.String
+import Prelude
+
+import Duckling.Locale
+import Duckling.Numeral.Types
+import Duckling.Resolve
+import Duckling.Testing.Types
+
+corpus :: Corpus
+corpus = (testContext {locale = makeLocale KN Nothing}, testOptions, allExamples)
+
+allExamples :: [Example]
+allExamples = concat
+  [ examples (NumeralValue 0)
+             [ "ಸೊನ್ನೆ"
+             , "೦"
+             ]
+  , examples (NumeralValue 1)
+             [ "ಒಂದು"
+             , "೧"
+             ]
+  , examples (NumeralValue 2)
+             [ "ಎರಡು"
+             , "೨"
+             ]
+  , examples (NumeralValue 3)
+             [ "ಮೂರು"
+             , "೩"
+             ]
+  , examples (NumeralValue 4)
+             [ "ನಾಲ್ಕು"
+             , "೪"
+             ]
+  , examples (NumeralValue 5)
+             [ "ಐದು"
+             , "೫"
+             ]
+  , examples (NumeralValue 6)
+             [ "ಆರು"
+             , "೬"
+             ]
+  , examples (NumeralValue 7)
+             [ "ಏಳು"
+             , "೭"
+             ]
+  , examples (NumeralValue 8)
+             [ "ಎಂಟು"
+             , "೮"
+             ]
+  , examples (NumeralValue 9)
+             [ "ಒಂಬತ್ತು"
+             , "೯"
+             ]
+  ]

Duckling/Numeral/KN/Rules.hs

@@ -0,0 +1,91 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE NoRebindableSyntax #-}
+
+module Duckling.Numeral.KN.Rules
+  ( rules
+  ) where
+
+import Data.HashMap.Strict (HashMap)
+import Data.Maybe
+import Data.Text (Text)
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.Text as Text
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.Helpers
+import Duckling.Regex.Types
+import Duckling.Types
+
+ankiMap :: HashMap Char Char
+ankiMap = HashMap.fromList
+  [ ('೦', '0')
+  , ('೧', '1')
+  , ('೨', '2')
+  , ('೩', '3')
+  , ('೪', '4')
+  , ('೫', '5')
+  , ('೬', '6')
+  , ('೭', '7')
+  , ('೮', '8')
+  , ('೯', '9')
+  ]
+
+ankiToArab :: Char -> Char
+ankiToArab c = HashMap.lookupDefault c c ankiMap
+
+ruleAnki :: Rule
+ruleAnki = Rule
+  { name = "anki forms"
+  , pattern =
+    [ regex "([೦೧೨೩೪೫೬೭೮೯]{1,10})"
+    ]
+  , prod = \case
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        toInteger <$> parseInt (Text.map ankiToArab match) >>= integer
+      _ -> Nothing
+  }
+
+ruleNumeralMap :: [(Text, Integer)]
+ruleNumeralMap =
+  [ ("ಸೊನ್ನೆ", 0)
+  , ("ಒಂದು", 1)
+  , ("ಎರಡು", 2)
+  , ("ಮೂರು", 3)
+  , ("ನಾಲ್ಕು", 4)
+  , ("ಐದು", 5)
+  , ("ಆರು", 6)
+  , ("ಏಳು", 7)
+  , ("ಎಂಟು", 8)
+  , ("ಒಂಬತ್ತು", 9)
+  ]
+
+ruleNumerals :: [Rule]
+ruleNumerals =
+  map constructRule ruleNumeralMap
+  where
+    constructRule :: (Text, Integer) -> Rule
+    constructRule (s, i) = Rule
+      { name = "number: " `mappend` s
+      , pattern =
+        [ regex $ Text.unpack s
+        ]
+      , prod = const $ integer i
+      }
+
+
+rules :: [Rule]
+rules =
+  [ ruleAnki
+  ]
+  ++ ruleNumerals

Duckling/Ranking/Classifiers.hs

@@ -39,6 +39,7 @@ import qualified Duckling.Ranking.Classifiers.IT_XX as IT_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.JA_XX as JA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KA_XX as KA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KM_XX as KM_XXClassifiers
+import qualified Duckling.Ranking.Classifiers.KN_XX as KN_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.KO_XX as KO_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.LO_XX as LO_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.ML_XX as ML_XXClassifiers
@@ -85,6 +86,7 @@ classifiers (Locale IT _) = IT_XXClassifiers.classifiers
 classifiers (Locale JA _) = JA_XXClassifiers.classifiers
 classifiers (Locale KA _) = KA_XXClassifiers.classifiers
 classifiers (Locale KM _) = KM_XXClassifiers.classifiers
+classifiers (Locale KN _) = KN_XXClassifiers.classifiers
 classifiers (Locale KO _) = KO_XXClassifiers.classifiers
 classifiers (Locale LO _) = LO_XXClassifiers.classifiers
 classifiers (Locale ML _) = ML_XXClassifiers.classifiers

Duckling/Ranking/Classifiers/IT_XX.hs

@@ -320,10 +320,10 @@ classifiers
                                unseen = -4.31748811353631,
                                likelihoods =
                                  HashMap.fromList
-                                   [("<integer> (latent time-of-day)", -0.9718605830289657),
+                                   [("<integer> (latent time-of-day)", -0.9718605830289658),
                                     ("intersect by \"di\", \"della\", \"del\"", -3.20545280453606),
                                     ("day", -2.3581549441488563), ("Lunedi", -3.6109179126442243),
-                                    ("hour", -0.9718605830289657),
+                                    ("hour", -0.9718605830289658),
                                     ("two time tokens separated by `di`", -3.20545280453606),
                                     ("Domenica", -3.6109179126442243)],
                                n = 33}}),

Duckling/Ranking/Classifiers/KN_XX.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+-----------------------------------------------------------------
+-- Auto-generated by regenClassifiers
+--
+-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+--  @generated
+-----------------------------------------------------------------
+{-# LANGUAGE OverloadedStrings #-}
+module Duckling.Ranking.Classifiers.KN_XX (classifiers) where
+import Data.String
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import Duckling.Ranking.Types
+
+classifiers :: Classifiers
+classifiers = HashMap.fromList []

Duckling/Rules.hs

@@ -46,6 +46,7 @@ import qualified Duckling.Rules.IT as ITRules
 import qualified Duckling.Rules.JA as JARules
 import qualified Duckling.Rules.KA as KARules
 import qualified Duckling.Rules.KM as KMRules
+import qualified Duckling.Rules.KN as KNRules
 import qualified Duckling.Rules.KO as KORules
 import qualified Duckling.Rules.LO as LORules
 import qualified Duckling.Rules.ML as MLRules
@@ -110,6 +111,7 @@ defaultRules IT = ITRules.defaultRules
 defaultRules JA = JARules.defaultRules
 defaultRules KA = KARules.defaultRules
 defaultRules KM = KMRules.defaultRules
+defaultRules KN = KNRules.defaultRules
 defaultRules KO = KORules.defaultRules
 defaultRules LO = LORules.defaultRules
 defaultRules ML = MLRules.defaultRules
@@ -154,6 +156,7 @@ localeRules IT = ITRules.localeRules
 localeRules JA = JARules.localeRules
 localeRules KA = KARules.localeRules
 localeRules KM = KMRules.localeRules
+localeRules KN = KNRules.localeRules
 localeRules KO = KORules.localeRules
 localeRules LO = LORules.localeRules
 localeRules ML = MLRules.localeRules
@@ -190,6 +193,7 @@ langRules FR = FRRules.langRules
 langRules GA = GARules.langRules
 langRules HE = HERules.langRules
 langRules HI = HIRules.langRules
+langRules KN = KNRules.langRules
 langRules HR = HRRules.langRules
 langRules HU = HURules.langRules
 langRules ID = IDRules.langRules

Duckling/Rules/KN.hs

@@ -0,0 +1,45 @@
+---Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+
+module Duckling.Rules.KN
+  ( defaultRules
+  , langRules
+  , localeRules
+  ) where
+
+import Duckling.Dimensions.Types
+import Duckling.Locale
+import Duckling.Types
+import qualified Duckling.Numeral.KN.Rules as Numeral
+
+defaultRules :: Some Dimension -> [Rule]
+defaultRules = langRules
+
+localeRules :: Region -> Some Dimension -> [Rule]
+localeRules region (This (CustomDimension dim)) = dimLocaleRules region dim
+localeRules _ _ = []
+
+langRules :: Some Dimension -> [Rule]
+langRules (This AmountOfMoney) = []
+langRules (This CreditCardNumber) = []
+langRules (This Distance) = []
+langRules (This Duration) = []
+langRules (This Numeral) = Numeral.rules
+langRules (This Email) = []
+langRules (This Ordinal) = []
+langRules (This PhoneNumber) = []
+langRules (This Quantity) = []
+langRules (This RegexMatch) = []
+langRules (This Temperature) = []
+langRules (This Time) = []
+langRules (This TimeGrain) = []
+langRules (This Url) = []
+langRules (This Volume) = []
+langRules (This (CustomDimension dim)) = dimLangRules KN dim

duckling.cabal

@@ -72,6 +72,7 @@ library
                      , Duckling.Rules.JA
                      , Duckling.Rules.KA
                      , Duckling.Rules.KM
+                     , Duckling.Rules.KN
                      , Duckling.Rules.KO
                      , Duckling.Rules.LO
                      , Duckling.Rules.ML
@@ -178,6 +179,7 @@ library
                      , Duckling.Dimensions.JA
                      , Duckling.Dimensions.KA
                      , Duckling.Dimensions.KM
+                     , Duckling.Dimensions.KN
                      , Duckling.Dimensions.KO
                      , Duckling.Dimensions.LO
                      , Duckling.Dimensions.ML
@@ -425,6 +427,8 @@ library
                      , Duckling.Numeral.KA.Rules
                      , Duckling.Numeral.KM.Corpus
                      , Duckling.Numeral.KM.Rules
+                     , Duckling.Numeral.KN.Corpus
+                     , Duckling.Numeral.KN.Rules
                      , Duckling.Numeral.KO.Corpus
                      , Duckling.Numeral.KO.Rules
                      , Duckling.Numeral.LO.Corpus
@@ -928,6 +932,7 @@ test-suite duckling-test
                      , Duckling.Numeral.JA.Tests
                      , Duckling.Numeral.KA.Tests
                      , Duckling.Numeral.KM.Tests
+                     , Duckling.Numeral.KN.Tests
                      , Duckling.Numeral.KO.Tests
                      , Duckling.Numeral.LO.Tests
                      , Duckling.Numeral.ML.Tests

exe/Duckling/Ranking/Generate.hs

@@ -193,6 +193,7 @@ getCorpusForLang IT = ITTime.corpus
 getCorpusForLang JA = (testContext, testOptions, [])
 getCorpusForLang KA = (testContext, testOptions, [])
 getCorpusForLang KM = (testContext, testOptions, [])
+getCorpusForLang KN = (testContext, testOptions, [])
 getCorpusForLang KO = KOTime.corpus
 getCorpusForLang LO = (testContext, testOptions, [])
 getCorpusForLang ML = (testContext, testOptions, [])

tests/Duckling/Numeral/KN/Tests.hs

@@ -0,0 +1,23 @@
+--Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+module Duckling.Numeral.KN.Tests
+  ( tests
+  ) where
+
+import Data.String
+import Prelude
+import Test.Tasty
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.KN.Corpus
+import Duckling.Testing.Asserts
+
+tests :: TestTree
+tests = testGroup "KN Tests"
+  [ makeCorpusTest [This Numeral] corpus
+  ]

tests/Duckling/Numeral/Tests.hs

@@ -35,6 +35,7 @@ import qualified Duckling.Numeral.IT.Tests as IT
 import qualified Duckling.Numeral.JA.Tests as JA
 import qualified Duckling.Numeral.KA.Tests as KA
 import qualified Duckling.Numeral.KM.Tests as KM
+import qualified Duckling.Numeral.KN.Tests as KN
 import qualified Duckling.Numeral.KO.Tests as KO
 import qualified Duckling.Numeral.LO.Tests as LO
 import qualified Duckling.Numeral.ML.Tests as ML
@@ -80,6 +81,7 @@ tests = testGroup "Numeral Tests"
   , JA.tests
   , KA.tests
   , KM.tests
+  , KN.tests
   , KO.tests
   , LO.tests
   , ML.tests