Basics for Finnish (#210)

Summary:
Adds Locale and Numeral for Finnish
Closes https://github.com/facebook/duckling/pull/210

Reviewed By: JonCoens

Differential Revision: D8430386

Pulled By: patapizza

fbshipit-source-id: a3c8b3b3419b7f43e2ef332cdb1fb8fc07da3bec

Duckling/Dimensions.hs

@@ -31,6 +31,7 @@ import qualified Duckling.Dimensions.EL as ELDimensions
 import qualified Duckling.Dimensions.EN as ENDimensions
 import qualified Duckling.Dimensions.ES as ESDimensions
 import qualified Duckling.Dimensions.ET as ETDimensions
+import qualified Duckling.Dimensions.FI as FIDimensions
 import qualified Duckling.Dimensions.FR as FRDimensions
 import qualified Duckling.Dimensions.GA as GADimensions
 import qualified Duckling.Dimensions.HE as HEDimensions
@@ -95,6 +96,7 @@ langDimensions EL = ELDimensions.allDimensions
 langDimensions EN = ENDimensions.allDimensions
 langDimensions ES = ESDimensions.allDimensions
 langDimensions ET = ETDimensions.allDimensions
+langDimensions FI = FIDimensions.allDimensions
 langDimensions FR = FRDimensions.allDimensions
 langDimensions GA = GADimensions.allDimensions
 langDimensions HE = HEDimensions.allDimensions

Duckling/Dimensions/FI.hs

@@ -0,0 +1,17 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+module Duckling.Dimensions.FI
+  ( allDimensions
+  ) where
+
+import Duckling.Dimensions.Types
+
+allDimensions :: [Some Dimension]
+allDimensions  =
+  [ This Numeral
+  ]

Duckling/Locale.hs

@@ -43,6 +43,7 @@ data Lang
   | EN
   | ES
   | ET
+  | FI
   | FR
   | GA
   | HE

Duckling/Numeral/FI/Corpus.hs

@@ -0,0 +1,123 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Duckling.Numeral.FI.Corpus
+  ( corpus
+  ) where
+
+import Data.String
+import Prelude
+
+import Duckling.Locale
+import Duckling.Numeral.Types
+import Duckling.Resolve
+import Duckling.Testing.Types
+
+context :: Context
+context = testContext {locale = makeLocale FI Nothing}
+
+corpus :: Corpus
+corpus = (context, testOptions, allExamples)
+
+allExamples :: [Example]
+allExamples = concat
+  [ examples (NumeralValue 0)
+             [ "0"
+             , "nolla"
+             ]
+  , examples (NumeralValue 1)
+             [ "1"
+             , "yksi"
+             ]
+  , examples (NumeralValue 2)
+             [ "kaksi"
+             ]
+  , examples (NumeralValue 3)
+             [ "kolme"
+             ]
+  , examples (NumeralValue 4)
+             [ "neljä"
+             ]
+  , examples (NumeralValue 5)
+             [ "viisi"
+             ]
+  , examples (NumeralValue 6)
+             [ "kuusi"
+             ]
+  , examples (NumeralValue 7)
+             [ "seitsemän"
+             ]
+  , examples (NumeralValue 8)
+             [ "kahdeksan"
+             ]
+  , examples (NumeralValue 9)
+             [ "yhdeksän"
+             ]
+  , examples (NumeralValue 10)
+             [ "10"
+             , "kymmenen"
+             ]
+  , examples (NumeralValue 11)
+             [ "yksitoista"
+             ]
+  , examples (NumeralValue 12)
+             [ "kaksitoista"
+             ]
+  , examples (NumeralValue 13)
+             [ "kolmetoista"
+             ]
+  , examples (NumeralValue 14)
+             [ "neljätoista"
+             ]
+  , examples (NumeralValue 15)
+             [ "viisitoista"
+             ]
+  , examples (NumeralValue 16)
+             [ "kuusitoista"
+             ]
+  , examples (NumeralValue 17)
+             [ "seitsemäntoista"
+             ]
+  , examples (NumeralValue 18)
+             [ "kahdeksantoista"
+             ]
+  , examples (NumeralValue 19)
+             [ "yhdeksäntoista"
+             ]
+  , examples (NumeralValue 20)
+             [ "20"
+             , "kaksikymmentä"
+             ]
+  , examples (NumeralValue 24)
+             [ "24"
+             , "kaksikymmentäneljä"
+             ]
+  , examples (NumeralValue 35)
+             [ "kolmekymmentäviisi"
+             ]
+  , examples (NumeralValue 42)
+             [ "neljäkymmentäkaksi"
+             ]
+  , examples (NumeralValue 52)
+             [ "viisikymmentäkaksi"
+             ]
+  , examples (NumeralValue 62)
+             [ "kuusikymmentäkaksi"
+             ]
+  , examples (NumeralValue 72)
+             [ "seitsemänkymmentäkaksi"
+             ]
+  , examples (NumeralValue 82)
+             [ "kahdeksankymmentäkaksi"
+             ]
+  , examples (NumeralValue 99)
+             [ "99"
+             , "yhdeksänkymmentäyhdeksän"
+             ]
+  ]

Duckling/Numeral/FI/Rules.hs

@@ -0,0 +1,142 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE NoRebindableSyntax #-}
+
+module Duckling.Numeral.FI.Rules
+  ( rules
+  ) where
+
+import Data.HashMap.Strict (HashMap)
+import Data.Maybe
+import Data.String
+import Data.Text (Text)
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.Text as Text
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.Helpers
+import Duckling.Numeral.Types (NumeralData(..))
+import Duckling.Regex.Types
+import Duckling.Types
+import qualified Duckling.Numeral.Types as TNumeral
+
+ruleIntegerNumeric :: Rule
+ruleIntegerNumeric = Rule
+  { name = "integer (numeric)"
+  , pattern =
+    [ regex "(\\d{1,18})"
+    ]
+  , prod = \case
+      (Token RegexMatch (GroupMatch (match:_)):_) -> do
+        v <- parseInt match
+        integer $ toInteger v
+      _ -> Nothing
+  }
+
+numeralMap :: HashMap Text Integer
+numeralMap = HashMap.fromList
+  [ ( "nolla", 0 )
+  , ( "yksi", 1 )
+  , ( "kaksi", 2 )
+  , ( "kolme", 3 )
+  , ( "neljä", 4 )
+  , ( "viisi", 5 )
+  , ( "kuusi", 6 )
+  , ( "seitsemän", 7 )
+  , ( "kahdeksan", 8 )
+  , ( "yhdeksän", 9 )
+  , ( "kymmenen", 10 )
+  ]
+
+ruleNumeral :: Rule
+ruleNumeral = Rule
+  { name = "number (0..10)"
+  , pattern =
+    [ regex "(nolla|yksi|kaksi|kolme|neljä|viisi|kuusi|seitsemän|kahdeksan|yhdeksän|kymmenen)"
+    ]
+  , prod = \case
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) numeralMap >>= integer
+      _ -> Nothing
+  }
+
+elevenToNineteenMap :: HashMap Text Integer
+elevenToNineteenMap = HashMap.fromList
+  [ ( "yksitoista", 11 )
+  , ( "kaksitoista", 12 )
+  , ( "kolmetoista", 13 )
+  , ( "neljätoista", 14 )
+  , ( "viisitoista", 15 )
+  , ( "kuusitoista", 16 )
+  , ( "seitsemäntoista", 17 )
+  , ( "kahdeksantoista", 18 )
+  , ( "yhdeksäntoista", 19 )
+  ]
+
+ruleElevenToNineteen :: Rule
+ruleElevenToNineteen = Rule
+  { name = "number (11..19)"
+  , pattern =
+    [ regex "(yksitoista|kaksitoista|kolmetoista|neljätoista|viisitoista|kuusitoista|seitsemäntoista|kahdeksantoista|yhdeksäntoista)"
+    ]
+  , prod = \case
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) elevenToNineteenMap >>= integer
+      _ -> Nothing
+  }
+
+tensMap :: HashMap Text Integer
+tensMap = HashMap.fromList
+  [ ( "kaksikymmentä", 20 )
+  , ( "kolmekymmentä", 30 )
+  , ( "neljäkymmentä", 40 )
+  , ( "viisikymmentä", 50 )
+  , ( "kuusikymmentä", 60 )
+  , ( "seitsemänkymmentä", 70 )
+  , ( "kahdeksankymmentä", 80 )
+  , ( "yhdeksänkymmentä", 90 )
+  ]
+
+ruleTens :: Rule
+ruleTens = Rule
+  { name = "integer (20,30..90)"
+  , pattern =
+    [ regex "(kaksikymmentä|kolmekymmentä|neljäkymmentä|viisikymmentä|kuusikymmentä|seitsemänkymmentä|kahdeksankymmentä|yhdeksänkymmentä)"
+    ]
+  , prod = \case
+      (Token RegexMatch (GroupMatch (match:_)):_) ->
+        HashMap.lookup (Text.toLower match) tensMap >>= integer
+      _ -> Nothing
+  }
+
+ruleCompositeTens :: Rule
+ruleCompositeTens = Rule
+  { name = "integer ([2-9][1-9])"
+  , pattern =
+    [ regex "(kaksikymmentä|kolmekymmentä|neljäkymmentä|viisikymmentä|kuusikymmentä|seitsemänkymmentä|kahdeksankymmentä|yhdeksänkymmentä)(yksi|kaksi|kolme|neljä|viisi|kuusi|seitsemän|kahdeksan|yhdeksän)"
+    ]
+  , prod = \case
+      (Token RegexMatch (GroupMatch (m1:m2:_)):_) -> do
+        v1 <- HashMap.lookup (Text.toLower m1) tensMap
+        v2 <- HashMap.lookup (Text.toLower m2) numeralMap
+        integer $ v1 + v2
+      _ -> Nothing
+  }
+
+rules :: [Rule]
+rules =
+  [ ruleIntegerNumeric
+  , ruleNumeral
+  , ruleElevenToNineteen
+  , ruleTens
+  , ruleCompositeTens
+  ]

Duckling/Ranking/Classifiers.hs

@@ -25,6 +25,7 @@ import qualified Duckling.Ranking.Classifiers.EN_US as EN_USClassifiers
 import qualified Duckling.Ranking.Classifiers.EN_XX as EN_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.ES_XX as ES_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.ET_XX as ET_XXClassifiers
+import qualified Duckling.Ranking.Classifiers.FI_XX as FI_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.FR_XX as FR_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.GA_XX as GA_XXClassifiers
 import qualified Duckling.Ranking.Classifiers.HE_XX as HE_XXClassifiers
@@ -63,6 +64,7 @@ classifiers (Locale EN (Just US)) = EN_USClassifiers.classifiers
 classifiers (Locale EN _) = EN_XXClassifiers.classifiers
 classifiers (Locale ES _) = ES_XXClassifiers.classifiers
 classifiers (Locale ET _) = ET_XXClassifiers.classifiers
+classifiers (Locale FI _) = FI_XXClassifiers.classifiers
 classifiers (Locale FR _) = FR_XXClassifiers.classifiers
 classifiers (Locale GA _) = GA_XXClassifiers.classifiers
 classifiers (Locale HE _) = HE_XXClassifiers.classifiers

Duckling/Ranking/Classifiers/FI_XX.hs

@@ -0,0 +1,22 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+-----------------------------------------------------------------
+-- Auto-generated by regenClassifiers
+--
+-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+--  @generated
+-----------------------------------------------------------------
+{-# LANGUAGE OverloadedStrings #-}
+module Duckling.Ranking.Classifiers.FI_XX (classifiers) where
+import Data.String
+import Prelude
+import qualified Data.HashMap.Strict as HashMap
+import Duckling.Ranking.Types
+
+classifiers :: Classifiers
+classifiers = HashMap.fromList []

Duckling/Rules.hs

@@ -32,6 +32,7 @@ import qualified Duckling.Rules.EL as ELRules
 import qualified Duckling.Rules.EN as ENRules
 import qualified Duckling.Rules.ES as ESRules
 import qualified Duckling.Rules.ET as ETRules
+import qualified Duckling.Rules.FI as FIRules
 import qualified Duckling.Rules.FR as FRRules
 import qualified Duckling.Rules.GA as GARules
 import qualified Duckling.Rules.HE as HERules
@@ -88,6 +89,7 @@ defaultRules EL = ELRules.defaultRules
 defaultRules EN = ENRules.defaultRules
 defaultRules ES = ESRules.defaultRules
 defaultRules ET = ETRules.defaultRules
+defaultRules FI = FIRules.defaultRules
 defaultRules FR = FRRules.defaultRules
 defaultRules GA = GARules.defaultRules
 defaultRules HE = HERules.defaultRules
@@ -124,6 +126,7 @@ localeRules EL = ELRules.localeRules
 localeRules EN = ENRules.localeRules
 localeRules ES = ESRules.localeRules
 localeRules ET = ETRules.localeRules
+localeRules FI = FIRules.localeRules
 localeRules FR = FRRules.localeRules
 localeRules GA = GARules.localeRules
 localeRules HE = HERules.localeRules
@@ -160,6 +163,7 @@ langRules EL = ELRules.langRules
 langRules EN = ENRules.langRules
 langRules ES = ESRules.langRules
 langRules ET = ETRules.langRules
+langRules FI = FIRules.langRules
 langRules FR = FRRules.langRules
 langRules GA = GARules.langRules
 langRules HE = HERules.langRules

Duckling/Rules/FI.hs

@@ -0,0 +1,44 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+{-# LANGUAGE GADTs #-}
+
+module Duckling.Rules.FI
+  ( defaultRules
+  , langRules
+  , localeRules
+  ) where
+
+import Duckling.Dimensions.Types
+import Duckling.Locale
+import Duckling.Types
+import qualified Duckling.Numeral.FI.Rules as Numeral
+import qualified Duckling.TimeGrain.FI.Rules as TimeGrain
+
+defaultRules :: Some Dimension -> [Rule]
+defaultRules = langRules
+
+localeRules :: Region -> Some Dimension -> [Rule]
+localeRules region (This (CustomDimension dim)) = dimLocaleRules region dim
+localeRules _ _ = []
+
+langRules :: Some Dimension -> [Rule]
+langRules (This AmountOfMoney) = []
+langRules (This Distance) = []
+langRules (This Duration) = []
+langRules (This Email) = []
+langRules (This Numeral) = Numeral.rules
+langRules (This Ordinal) = []
+langRules (This PhoneNumber) = []
+langRules (This Quantity) = []
+langRules (This RegexMatch) = []
+langRules (This Temperature) = []
+langRules (This Time) = []
+langRules (This TimeGrain) = TimeGrain.rules
+langRules (This Url) = []
+langRules (This Volume) = []
+langRules (This (CustomDimension dim)) = dimLangRules FI dim

Duckling/TimeGrain/FI/Rules.hs

@@ -0,0 +1,41 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+
+{-# LANGUAGE OverloadedStrings #-}
+
+module Duckling.TimeGrain.FI.Rules
+  ( rules
+  ) where
+
+import Data.String
+import Data.Text (Text)
+import Prelude
+
+import Duckling.Dimensions.Types
+import Duckling.Types
+import qualified Duckling.TimeGrain.Types as TG
+
+grains :: [(Text, String, TG.Grain)]
+grains = [ ("second (grain) ", "sekuntia?|s", TG.Second)
+         , ("minute (grain)" , "minuuttia?|min", TG.Minute)
+         , ("hour (grain)"   , "tuntia?|h", TG.Hour)
+         , ("day (grain)"    , "päivä(ä)?|vuorokau(si|tta)|vrk|d", TG.Day)
+         , ("week (grain)"   , "viikkoa?|vko?", TG.Week)
+         , ("month (grain)"  , "kuukau(si|tta)|kk", TG.Month)
+         , ("quarter (grain)", "kvartaali(a)?|neljännesvuo(si|tta)", TG.Quarter)
+         , ("year (grain)"   , "vuo(si|tta)|v\\.?|a", TG.Year)
+         ]
+
+rules :: [Rule]
+rules = map go grains
+  where
+    go (name, regexPattern, grain) = Rule
+      { name = name
+      , pattern = [regex regexPattern]
+      , prod = \_ -> Just $ Token TimeGrain grain
+      }

duckling.cabal

@@ -50,6 +50,7 @@ library
                      , Duckling.Rules.EN
                      , Duckling.Rules.ES
                      , Duckling.Rules.ET
+                     , Duckling.Rules.FI
                      , Duckling.Rules.FR
                      , Duckling.Rules.GA
                      , Duckling.Rules.HE
@@ -94,6 +95,7 @@ library
                      , Duckling.Ranking.Classifiers.EN_XX
                      , Duckling.Ranking.Classifiers.ES_XX
                      , Duckling.Ranking.Classifiers.ET_XX
+                     , Duckling.Ranking.Classifiers.FI_XX
                      , Duckling.Ranking.Classifiers.FR_XX
                      , Duckling.Ranking.Classifiers.GA_XX
                      , Duckling.Ranking.Classifiers.HE_XX
@@ -140,6 +142,7 @@ library
                      , Duckling.Dimensions.EN
                      , Duckling.Dimensions.ES
                      , Duckling.Dimensions.ET
+                     , Duckling.Dimensions.FI
                      , Duckling.Dimensions.FR
                      , Duckling.Dimensions.GA
                      , Duckling.Dimensions.HE
@@ -339,6 +342,8 @@ library
                      , Duckling.Numeral.ES.Rules
                      , Duckling.Numeral.ET.Corpus
                      , Duckling.Numeral.ET.Rules
+                     , Duckling.Numeral.FI.Corpus
+                     , Duckling.Numeral.FI.Rules
                      , Duckling.Numeral.FR.Corpus
                      , Duckling.Numeral.FR.Rules
                      , Duckling.Numeral.GA.Corpus
@@ -613,6 +618,7 @@ library
                      , Duckling.TimeGrain.EN.Rules
                      , Duckling.TimeGrain.EL.Rules
                      , Duckling.TimeGrain.ES.Rules
+                     , Duckling.TimeGrain.FI.Rules
                      , Duckling.TimeGrain.FR.Rules
                      , Duckling.TimeGrain.GA.Rules
                      , Duckling.TimeGrain.HE.Rules
@@ -788,6 +794,7 @@ test-suite duckling-test
                      , Duckling.Numeral.EN.Tests
                      , Duckling.Numeral.ES.Tests
                      , Duckling.Numeral.ET.Tests
+                     , Duckling.Numeral.FI.Tests
                      , Duckling.Numeral.FR.Tests
                      , Duckling.Numeral.GA.Tests
                      , Duckling.Numeral.HE.Tests

exe/Duckling/Ranking/Generate.hs

@@ -171,6 +171,7 @@ getCorpusForLang EL = ELTime.corpus
 getCorpusForLang EN = ENTime.corpus
 getCorpusForLang ES = ESTime.corpus
 getCorpusForLang ET = (testContext, testOptions, [])
+getCorpusForLang FI = (testContext, testOptions, [])
 getCorpusForLang FR = FRTime.corpus
 getCorpusForLang GA = GATime.corpus
 getCorpusForLang HR = HRTime.corpus

tests/Duckling/Numeral/FI/Tests.hs

@@ -0,0 +1,23 @@
+-- Copyright (c) 2016-present, Facebook, Inc.
+-- All rights reserved.
+--
+-- This source code is licensed under the BSD-style license found in the
+-- LICENSE file in the root directory of this source tree. An additional grant
+-- of patent rights can be found in the PATENTS file in the same directory.
+
+module Duckling.Numeral.FI.Tests
+  ( tests
+  ) where
+
+import Data.String
+import Prelude
+import Test.Tasty
+
+import Duckling.Dimensions.Types
+import Duckling.Numeral.FI.Corpus
+import Duckling.Testing.Asserts
+
+tests :: TestTree
+tests = testGroup "FI Tests"
+  [ makeCorpusTest [This Numeral] corpus
+  ]

tests/Duckling/Numeral/Tests.hs

@@ -21,6 +21,7 @@ import qualified Duckling.Numeral.EL.Tests as EL
 import qualified Duckling.Numeral.EN.Tests as EN
 import qualified Duckling.Numeral.ES.Tests as ES
 import qualified Duckling.Numeral.ET.Tests as ET
+import qualified Duckling.Numeral.FI.Tests as FI
 import qualified Duckling.Numeral.FR.Tests as FR
 import qualified Duckling.Numeral.GA.Tests as GA
 import qualified Duckling.Numeral.HE.Tests as HE
@@ -59,6 +60,7 @@ tests = testGroup "Numeral Tests"
   , ES.tests
   , ET.tests
   , FR.tests
+  , FI.tests
   , GA.tests
   , HE.tests
   , HI.tests