Hindi Language Numeral Dimension(minimalistic model). Tests passed.

Summary: Closes https://github.com/facebook/duckling/pull/119

Reviewed By: JonCoens

Differential Revision: D6597628

Pulled By: patapizza

fbshipit-source-id: 8bac0f686d6cecc38d9998e37042fe48f73530dc

.gitignore

@@ -1,2 +1,3 @@
 .stack-work/
 log/
+.idea/

Duckling/Dimensions.hs

@@ -32,6 +32,7 @@ import qualified Duckling.Dimensions.ET as ETDimensions
 import qualified Duckling.Dimensions.FR as FRDimensions
 import qualified Duckling.Dimensions.GA as GADimensions
 import qualified Duckling.Dimensions.HE as HEDimensions
+import qualified Duckling.Dimensions.HI as HIDimensions
 import qualified Duckling.Dimensions.HR as HRDimensions
 import qualified Duckling.Dimensions.HU as HUDimensions
 import qualified Duckling.Dimensions.ID as IDDimensions
@@ -93,6 +94,7 @@ langDimensions ET = ETDimensions.allDimensions
 langDimensions FR = FRDimensions.allDimensions
 langDimensions GA = GADimensions.allDimensions
 langDimensions HE = HEDimensions.allDimensions
+langDimensions HI = HIDimensions.allDimensions
 langDimensions HR = HRDimensions.allDimensions
 langDimensions HU = HUDimensions.allDimensions
 langDimensions ID = IDDimensions.allDimensions

Duckling/Dimensions/HI.hs

@@ -0,0 +1,18 @@
+--Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+module Duckling.Dimensions.HI
+  ( allDimensions
+  ) where
+
+import Duckling.Dimensions.Types
+
+allDimensions :: [Some Dimension]
+allDimensions =
+  [ This Numeral
+  ]

Duckling/Locale.hs

@@ -43,6 +43,7 @@ data Lang
   | FR
   | GA
   | HE
+  | HI
   | HR
   | HU
   | ID

Duckling/Numeral/HI/Corpus.hs

@@ -0,0 +1,87 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Duckling.Numeral.HI.Corpus
+  ( corpus
+  ) where
+
+import Data.String
+import Prelude
+
+import Duckling.Locale
+import Duckling.Numeral.Types
+import Duckling.Resolve
+import Duckling.Testing.Types
+
+corpus :: Corpus
+corpus = (testContext {locale = makeLocale HI Nothing}, allExamples)
+
+allExamples :: [Example]
+allExamples = concat
+  [ examples (NumeralValue 0)
+             [ "शून्य"
+             , "०"
+             ]
+  , examples (NumeralValue 1)
+             [ "एक"
+             ]
+  , examples (NumeralValue 2)
+             [ "दो"
+             ]
+  , examples (NumeralValue 3)
+             [ "तीन"
+             ]
+  , examples (NumeralValue 4)
+             [ "चार"
+             ]
+  , examples (NumeralValue 5)
+             [ "पाँच"
+             ]
+  , examples (NumeralValue 6)
+             [ "छह"
+             ]
+  , examples (NumeralValue 7)
+             [ "सात"
+             ]
+  , examples (NumeralValue 8)
+             [ "आठ"
+             ]
+  , examples (NumeralValue 9)
+             [ "नौ"
+             ]
+  , examples (NumeralValue 10)
+             [ "दस"
+             ]
+  , examples (NumeralValue 15)
+             [ "पन्द्रह"
+             ]
+  , examples (NumeralValue 17)
+             [ "सत्रह"
+             ]
+  , examples (NumeralValue 20)
+             [ "बीस"
+             ]
+  , examples (NumeralValue 22)
+             [ "बाईस"
+             ]
+  , examples (NumeralValue 24)
+             [ "चौबीस"
+             ]
+  , examples (NumeralValue 26)
+             [ "छब्बीस"
+             ]
+  , examples (NumeralValue 28)
+             [ "अट्ठाईस"
+             ]
+  , examples (NumeralValue 50)
+             [ "५०"
+             , "पचास"
+             ]
+  ]

Duckling/Numeral/HI/Rules.hs

@@ -0,0 +1,170 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE NoRebindableSyntax #-}
+
+module Duckling.Numeral.HI.Rules
+  ( rules
+  ) where
+
+import Data.HashMap.Strict (HashMap)
+import Data.Maybe
+import Data.String
+import Data.Text (Text)
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.Text as Text
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.Helpers
+import Duckling.Numeral.Types (NumeralData (..))
+import Duckling.Regex.Types
+import Duckling.Types
+import qualified Duckling.Numeral.Types as TNumeral
+
+devanagariMap :: HashMap Char Char
+devanagariMap = HashMap.fromList
+  [ ( '०', '0' )
+  , ( '१', '1' )
+  , ( '२', '2' )
+  , ( '३', '3' )
+  , ( '४', '4' )
+  , ( '५', '5' )
+  , ( '६', '6' )
+  , ( '७', '7' )
+  , ( '८', '8' )
+  , ( '९', '9' )
+  ]
+
+devanagariToArab :: Char -> Char
+devanagariToArab c = HashMap.lookupDefault c c devanagariMap
+
+ruleDevanagari :: Rule
+ruleDevanagari = Rule
+  { name = "devanagari forms"
+  , pattern =
+    [ regex "([०१२३४५६७८९]{1,18})"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        toInteger <$> parseInt (Text.map devanagariToArab match) >>= integer
+      _ -> Nothing
+  }
+
+ruleNumeralMap :: HashMap Text Integer
+ruleNumeralMap = HashMap.fromList
+  [ ( "शून्य", 0 )
+  , ( "एक", 1 )
+  , ( "दो" , 2 )
+  , ( "तीन", 3 )
+  , ( "चार", 4 )
+  , ( "पाँच", 5 )
+  , ( "छह", 6 )
+  , ( "सात", 7 )
+  , ( "आठ", 8 )
+  , ( "नौ" , 9 )
+  , ( "दस", 10 )
+  ]
+
+ruleNumeral :: Rule
+ruleNumeral = Rule
+  { name = "number (0..10)"
+  , pattern =
+    [ regex "(शून्य|एक|दो|तीन|चार|पाँच|छह|सात|आठ|नौ|दस)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup match ruleNumeralMap >>= integer
+      _ -> Nothing
+  }
+
+elevenToNineteenMap :: HashMap Text Integer
+elevenToNineteenMap = HashMap.fromList
+  [ ( "ग्यारह", 11 )
+  , ( "बारह", 12 )
+  , ( "तेरह", 13 )
+  , ( "चौदह", 14 )
+  , ( "पन्द्रह", 15 )
+  , ( "सोलह", 16 )
+  , ( "सत्रह", 17 )
+  , ( "अठारह", 18 )
+  , ( "उन्नीस", 19 )
+  ]
+
+ruleElevenToNineteen :: Rule
+ruleElevenToNineteen = Rule
+  { name = "number (11..19)"
+  , pattern =
+    [ regex "(ग्यारह|बारह|तेरह|चौदह|पन्द्रह|सोलह|सत्रह|अठारह|उन्नीस)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup match elevenToNineteenMap >>= integer
+      _ -> Nothing
+  }
+
+twentyoneToTwentynineMap :: HashMap Text Integer
+twentyoneToTwentynineMap = HashMap.fromList
+  [ ( "इक्कीस", 21 )
+  , ( "बाईस", 22 )
+  , ( "तेईस", 23 )
+  , ( "चौबीस", 24 )
+  , ( "पच्चीस", 25 )
+  , ( "छब्बीस", 26 )
+  , ( "सत्ताईस", 27 )
+  , ( "अट्ठाईस", 28 )
+  , ( "उनतीस", 29 )
+  ]
+
+ruleTwentyoneToTwentynine :: Rule
+ruleTwentyoneToTwentynine = Rule
+  { name = "number (21..29)"
+  , pattern =
+    [ regex "(इक्कीस|बाईस|तेईस|चौबीस|पच्चीस|छब्बीस|सत्ताईस|अट्ठाईस|उनतीस)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup match twentyoneToTwentynineMap >>= integer
+      _ -> Nothing
+  }
+
+tensMap :: HashMap Text Integer
+tensMap = HashMap.fromList
+  [ ( "बीस", 20 )
+  , ( "तीस", 30 )
+  , ( "चालीस", 40 )
+  , ( "पचास", 50 )
+  , ( "साठ", 60 )
+  , ( "सत्तर", 70 )
+  , ( "अस्सी", 80 )
+  , ( "नब्बे", 90 )
+  ]
+
+ruleTens :: Rule
+ruleTens = Rule
+  { name = "integer (20,30..90)"
+  , pattern =
+    [ regex "(बीस|तीस|चालीस|पचास|साठ|सत्तर|अस्सी|नब्बे)"
+    ]
+  , prod = \tokens -> case tokens of
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup match tensMap >>= integer
+      _ -> Nothing
+  }
+
+
+rules :: [Rule]
+rules =
+  [ ruleDevanagari
+  , ruleNumeral
+  , ruleElevenToNineteen
+  , ruleTwentyoneToTwentynine
+  , ruleTens
+  ]

Duckling/Ranking/Classifiers.hs

@@ -28,6 +28,7 @@ import qualified Duckling.Ranking.Classifiers.ET_XX as ET_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.FR_XX as FR_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.GA_XX as GA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.HE_XX as HE_XXClassifiers
+import qualified Duckling.Ranking.Classifiers.HI_XX as HI_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.HR_XX as HR_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.HU_XX as HU_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.ID_XX as ID_XXClassifiers
@@ -63,6 +64,7 @@ classifiers (Locale ET _) = ET_XXClassifiers.classifiers
 classifiers (Locale FR _) = FR_XXClassifiers.classifiers
 classifiers (Locale GA _) = GA_XXClassifiers.classifiers
 classifiers (Locale HE _) = HE_XXClassifiers.classifiers
+classifiers (Locale HI _) = HI_XXClassifiers.classifiers
 classifiers (Locale HR _) = HR_XXClassifiers.classifiers
 classifiers (Locale HU _) = HU_XXClassifiers.classifiers
 classifiers (Locale ID _) = ID_XXClassifiers.classifiers

Duckling/Ranking/Classifiers/HI_XX.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+-----------------------------------------------------------------
+-- Auto-generated by regenClassifiers
+--
+-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+--  @generated
+-----------------------------------------------------------------
+{-# LANGUAGE OverloadedStrings #-}
+module Duckling.Ranking.Classifiers.HI_XX (classifiers) where
+import Data.String
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import Duckling.Ranking.Types
+
+classifiers :: Classifiers
+classifiers = HashMap.fromList []

Duckling/Rules.hs

@@ -35,6 +35,7 @@ import qualified Duckling.Rules.ET as ETRules
 import qualified Duckling.Rules.FR as FRRules
 import qualified Duckling.Rules.GA as GARules
 import qualified Duckling.Rules.HE as HERules
+import qualified Duckling.Rules.HI as HIRules
 import qualified Duckling.Rules.HR as HRRules
 import qualified Duckling.Rules.HU as HURules
 import qualified Duckling.Rules.ID as IDRules
@@ -88,6 +89,7 @@ defaultRules ET = ETRules.defaultRules
 defaultRules FR = FRRules.defaultRules
 defaultRules GA = GARules.defaultRules
 defaultRules HE = HERules.defaultRules
+defaultRules HI = HIRules.defaultRules
 defaultRules HR = HRRules.defaultRules
 defaultRules HU = HURules.defaultRules
 defaultRules ID = IDRules.defaultRules
@@ -121,6 +123,7 @@ localeRules ET = ETRules.localeRules
 localeRules FR = FRRules.localeRules
 localeRules GA = GARules.localeRules
 localeRules HE = HERules.localeRules
+localeRules HI = HIRules.localeRules
 localeRules HR = HRRules.localeRules
 localeRules HU = HURules.localeRules
 localeRules ID = IDRules.localeRules
@@ -154,6 +157,7 @@ langRules ET = ETRules.langRules
 langRules FR = FRRules.langRules
 langRules GA = GARules.langRules
 langRules HE = HERules.langRules
+langRules HI = HIRules.langRules
 langRules HR = HRRules.langRules
 langRules HU = HURules.langRules
 langRules ID = IDRules.langRules

Duckling/Rules/HI.hs

@@ -0,0 +1,42 @@
+--Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE GADTs #-}
+
+module Duckling.Rules.HI
+  ( defaultRules
+  , langRules
+  , localeRules
+  ) where
+
+import Duckling.Dimensions.Types
+import Duckling.Locale
+import Duckling.Types
+import qualified Duckling.Numeral.HI.Rules as Numeral
+
+defaultRules :: Some Dimension -> [Rule]
+defaultRules = langRules
+
+localeRules :: Region -> Some Dimension -> [Rule]
+localeRules _ _ = []
+
+langRules :: Some Dimension -> [Rule]
+langRules (This AmountOfMoney) = []
+langRules (This Distance) = []
+langRules (This Duration) = []
+langRules (This Numeral) = Numeral.rules
+langRules (This Email) = []
+langRules (This Ordinal) = []
+langRules (This PhoneNumber) = []
+langRules (This Quantity) = []
+langRules (This RegexMatch) = []
+langRules (This Temperature) = []
+langRules (This Time) = []
+langRules (This TimeGrain) = []
+langRules (This Url) = []
+langRules (This Volume) = []

duckling.cabal

@@ -52,6 +52,7 @@ library
                      , Duckling.Rules.FR
                      , Duckling.Rules.GA
                      , Duckling.Rules.HE
+                     , Duckling.Rules.HI
                      , Duckling.Rules.HR
                      , Duckling.Rules.HU
                      , Duckling.Rules.ID
@@ -93,6 +94,7 @@ library
                      , Duckling.Ranking.Classifiers.FR_XX
                      , Duckling.Ranking.Classifiers.GA_XX
                      , Duckling.Ranking.Classifiers.HE_XX
+                     , Duckling.Ranking.Classifiers.HI_XX
                      , Duckling.Ranking.Classifiers.HR_XX
                      , Duckling.Ranking.Classifiers.HU_XX
                      , Duckling.Ranking.Classifiers.ID_XX
@@ -134,6 +136,7 @@ library
                      , Duckling.Dimensions.FR
                      , Duckling.Dimensions.GA
                      , Duckling.Dimensions.HE
+                     , Duckling.Dimensions.HI
                      , Duckling.Dimensions.HR
                      , Duckling.Dimensions.HU
                      , Duckling.Dimensions.ID
@@ -293,6 +296,8 @@ library
                      , Duckling.Numeral.GA.Rules
                      , Duckling.Numeral.HE.Corpus
                      , Duckling.Numeral.HE.Rules
+                     , Duckling.Numeral.HI.Corpus
+                     , Duckling.Numeral.HI.Rules
                      , Duckling.Numeral.HR.Corpus
                      , Duckling.Numeral.HR.Rules
                      , Duckling.Numeral.HU.Corpus
@@ -703,6 +708,7 @@ test-suite duckling-test
                      , Duckling.Numeral.FR.Tests
                      , Duckling.Numeral.GA.Tests
                      , Duckling.Numeral.HE.Tests
+                     , Duckling.Numeral.HI.Tests
                      , Duckling.Numeral.HR.Tests
                      , Duckling.Numeral.HU.Tests
                      , Duckling.Numeral.ID.Tests

exe/Duckling/Ranking/Generate.hs

@@ -173,6 +173,7 @@ getCorpusForLang GA = GATime.corpus
 getCorpusForLang HR = HRTime.corpus
 getCorpusForLang HE = HETime.corpus
 getCorpusForLang HU = HUTime.corpus
+getCorpusForLang HI = (testContext, [])
 getCorpusForLang ID = (testContext, [])
 getCorpusForLang IT = ITTime.corpus
 getCorpusForLang JA = (testContext, [])

tests/Duckling/Numeral/HI/Tests.hs

@@ -0,0 +1,23 @@
+--Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+module Duckling.Numeral.HI.Tests
+  ( tests
+  ) where
+
+import Data.String
+import Prelude
+import Test.Tasty
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.HI.Corpus
+import Duckling.Testing.Asserts
+
+tests :: TestTree
+tests = testGroup "HI Tests"
+  [ makeCorpusTest [This Numeral] corpus
+  ]

tests/Duckling/Numeral/Tests.hs

@@ -24,6 +24,7 @@ import qualified Duckling.Numeral.ET.Tests as ET
 import qualified Duckling.Numeral.FR.Tests as FR
 import qualified Duckling.Numeral.GA.Tests as GA
 import qualified Duckling.Numeral.HE.Tests as HE
+import qualified Duckling.Numeral.HI.Tests as HI
 import qualified Duckling.Numeral.HR.Tests as HR
 import qualified Duckling.Numeral.HU.Tests as HU
 import qualified Duckling.Numeral.ID.Tests as ID
@@ -58,6 +59,7 @@ tests = testGroup "Numeral Tests"
   , FR.tests
   , GA.tests
   , HE.tests
+  , HI.tests
   , HR.tests
   , HU.tests
   , ID.tests