Setup + Numeral

Summary: Setup + Numeral dimension for Croatian.

Reviewed By: niteria

Differential Revision: D4946964

fbshipit-source-id: 204429b
This commit is contained in:
Julien Odent 2017-04-26 09:11:03 -07:00 committed by Facebook Github Bot
parent c70cf6d38d
commit 840deda7dd
13 changed files with 622 additions and 0 deletions

View File

@ -29,6 +29,7 @@ import qualified Duckling.Dimensions.ET as ETDimensions
import qualified Duckling.Dimensions.FR as FRDimensions
import qualified Duckling.Dimensions.GA as GADimensions
import qualified Duckling.Dimensions.HE as HEDimensions
import qualified Duckling.Dimensions.HR as HRDimensions
import qualified Duckling.Dimensions.ID as IDDimensions
import qualified Duckling.Dimensions.IT as ITDimensions
import qualified Duckling.Dimensions.JA as JADimensions
@ -84,6 +85,7 @@ langDimensions ET = ETDimensions.allDimensions
langDimensions FR = FRDimensions.allDimensions
langDimensions GA = GADimensions.allDimensions
langDimensions HE = HEDimensions.allDimensions
langDimensions HR = HRDimensions.allDimensions
langDimensions ID = IDDimensions.allDimensions
langDimensions IT = ITDimensions.allDimensions
langDimensions JA = JADimensions.allDimensions

18
Duckling/Dimensions/HR.hs Normal file
View File

@ -0,0 +1,18 @@
-- Copyright (c) 2016-present, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
module Duckling.Dimensions.HR
( allDimensions
) where
import Duckling.Dimensions.Types
allDimensions :: [Some Dimension]
allDimensions =
[ This Numeral
]

View File

@ -31,6 +31,7 @@ data Lang
| FR
| GA
| HE
| HR
| ID
| IT
| JA

View File

@ -0,0 +1,127 @@
-- Copyright (c) 2016-present, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
{-# LANGUAGE OverloadedStrings #-}
module Duckling.Numeral.HR.Corpus
( corpus ) where
import Prelude
import Data.String
import Duckling.Lang
import Duckling.Numeral.Types
import Duckling.Resolve
import Duckling.Testing.Types
corpus :: Corpus
corpus = (testContext {lang = HR}, allExamples)
allExamples :: [Example]
allExamples = concat
[ examples (NumeralValue 0)
[ "0"
, "nula"
, "nista"
, "ništa"
, "nistica"
, "ništica"
]
, examples (NumeralValue 1)
[ "1"
, "jedan"
, "sam"
]
, examples (NumeralValue 2)
[ "2"
, "dva"
, "par"
]
, examples (NumeralValue 33)
[ "33"
, "trideset i tri"
, "trideset tri"
, "0033"
]
, examples (NumeralValue 14)
[ "14"
, "cetrnaest"
, "četrnaest"
]
, examples (NumeralValue 16)
[ "16"
, "šesnaest"
, "sesnaest"
]
, examples (NumeralValue 17)
[ "17"
, "sedamnaest"
]
, examples (NumeralValue 18)
[ "18"
, "osamnaest"
]
, examples (NumeralValue 1.1)
[ "1,1"
, "jedan cijela jedan"
, "1,10"
, "01,10"
]
, examples (NumeralValue 0.77)
[ "0,77"
, ",77"
]
, examples (NumeralValue 100000)
[ "100.000"
, "100000"
, "100K"
, "100k"
]
, examples (NumeralValue 3000000)
[ "3M"
, "3000K"
, "3000000"
, "3.000.000"
]
, examples (NumeralValue 1200000)
[ "1.200.000"
, "1200000"
, "1,2M"
, "1200K"
, ",0012G"
]
, examples (NumeralValue (-1200000))
[ "- 1.200.000"
, "-1200000"
, "minus 1.200.000"
, "-1,2M"
, "-1200K"
, "-,0012G"
]
, examples (NumeralValue 5000)
[ "5 tisuća"
, "pet tisuća"
, "pet tisuca"
]
, examples (NumeralValue 122)
[ "stotinu dvadeset dva"
]
, examples (NumeralValue 200000)
[ "dvjesto tisuća"
, "dvjesto tisuca"
, "dvije stotine tisuca"
, "dvije stotine tisuća"
]
, examples (NumeralValue 21011)
[ "dvadeset i jedna tisuca jedanaest"
, "dvadeset i jedna tisuća jedanaest"
]
, examples (NumeralValue 721012)
[ "sedam stotina dvadeset jedna tisuća dvanaest"
, "sedam stotina dvadeset jedna tisuca dvanaest"
]
]

View File

@ -0,0 +1,383 @@
-- Copyright (c) 2016-present, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
{-# LANGUAGE GADTs #-}
{-# LANGUAGE OverloadedStrings #-}
module Duckling.Numeral.HR.Rules
( rules ) where
import Data.Maybe
import Data.String
import Data.Text (Text)
import Prelude
import qualified Data.Text as Text
import Duckling.Dimensions.Types
import Duckling.Numeral.Helpers
import Duckling.Numeral.Types (NumeralData (..))
import Duckling.Regex.Types
import Duckling.Types
import qualified Duckling.Numeral.Types as TNumeral
ruleNumbersPrefixWithNegativeOrMinus :: Rule
ruleNumbersPrefixWithNegativeOrMinus = Rule
{ name = "numbers prefix with -, negative or minus"
, pattern =
[ regex "-|minus|negativ"
, dimension Numeral
]
, prod = \tokens -> case tokens of
(_:Token Numeral nd:_) -> double (TNumeral.value nd * (-1))
_ -> Nothing
}
ruleIntegerNumeric :: Rule
ruleIntegerNumeric = Rule
{ name = "integer (numeric)"
, pattern =
[ regex "(\\d{1,18})"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):_) -> do
v <- toInteger <$> parseInt match
integer v
_ -> Nothing
}
ruleFew :: Rule
ruleFew = Rule
{ name = "few"
, pattern =
[ regex "nekoliko"
]
, prod = \_ -> integer 3
}
ruleTen :: Rule
ruleTen = Rule
{ name = "ten"
, pattern =
[ regex "deset|cener"
]
, prod = \_ -> integer 10 >>= withGrain 1
}
ruleDecimalWithThousandsSeparator :: Rule
ruleDecimalWithThousandsSeparator = Rule
{ name = "decimal with thousands separator"
, pattern =
[ regex "(\\d+(\\.\\d\\d\\d)+\\,\\d+)"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):_) ->
let dot = Text.singleton '.'
comma = Text.singleton ','
fmt = Text.replace comma dot $ Text.replace dot Text.empty match
in parseDouble fmt >>= double
_ -> Nothing
}
ruleDecimalNumber :: Rule
ruleDecimalNumber = Rule
{ name = "decimal number"
, pattern =
[ regex "(\\d*,\\d+)"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):
_) -> parseDecimal False match
_ -> Nothing
}
ruleInteger3 :: Rule
ruleInteger3 = Rule
{ name = "integer (100..900)"
, pattern =
[ regex "(sto|dvjest(o|a)|tristo|(c|\x010d)etiristo|petsto|(\x0161|s)esto|sedamsto|osamsto|devetsto)"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):_) -> case Text.toLower match of
"sto" -> integer 100
"dvjesta" -> integer 200
"dvjesto" -> integer 200
"tristo" -> integer 300
"cetiristo" -> integer 400
"\269etiristo" -> integer 400
"petsto" -> integer 500
"\353esto" -> integer 600
"sesto" -> integer 600
"sedamsto" -> integer 700
"osamsto" -> integer 800
"devetsto" -> integer 900
_ -> Nothing
_ -> Nothing
}
ruleSingle :: Rule
ruleSingle = Rule
{ name = "single"
, pattern =
[ regex "sam"
]
, prod = \_ -> integer 1 >>= withGrain 1
}
rulePowersOfTen :: Rule
rulePowersOfTen = Rule
{ name = "powers of tens"
, pattern =
[ regex "(stotin(u|a|e)|tisu(c|\x0107)(a|u|e)|milij(u|o)na?)"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):_) -> case Text.toLower match of
"stotinu" -> double 1e2 >>= withGrain 2 >>= withMultipliable
"stotina" -> double 1e2 >>= withGrain 2 >>= withMultipliable
"stotine" -> double 1e2 >>= withGrain 2 >>= withMultipliable
"tisuca" -> double 1e3 >>= withGrain 3 >>= withMultipliable
"tisucu" -> double 1e3 >>= withGrain 3 >>= withMultipliable
"tisuce" -> double 1e3 >>= withGrain 3 >>= withMultipliable
"tisu\263a" -> double 1e3 >>= withGrain 3 >>= withMultipliable
"tisu\263u" -> double 1e3 >>= withGrain 3 >>= withMultipliable
"tisu\263e" -> double 1e3 >>= withGrain 3 >>= withMultipliable
"milijun" -> double 1e6 >>= withGrain 6 >>= withMultipliable
"milijuna" -> double 1e6 >>= withGrain 6 >>= withMultipliable
"milijon" -> double 1e6 >>= withGrain 6 >>= withMultipliable
"milijona" -> double 1e6 >>= withGrain 6 >>= withMultipliable
_ -> Nothing
_ -> Nothing
}
ruleNumbersI :: Rule
ruleNumbersI = Rule
{ name = "numbers i"
, pattern =
[ oneOf [70, 20, 60, 50, 40, 90, 30, 80]
, regex "i"
, numberBetween 1 10
]
, prod = \tokens -> case tokens of
(Token Numeral (NumeralData {TNumeral.value = v1}):
_:
Token Numeral (NumeralData {TNumeral.value = v2}):
_) -> double $ v1 + v2
_ -> Nothing
}
ruleSum :: Rule
ruleSum = Rule
{ name = "intersect"
, pattern =
[ numberWith (fromMaybe 0 . TNumeral.grain) (>1)
, numberWith TNumeral.multipliable not
]
, prod = \tokens -> case tokens of
(Token Numeral (NumeralData {TNumeral.value = val1, TNumeral.grain = Just g}):
Token Numeral (NumeralData {TNumeral.value = val2}):
_) | (10 ** fromIntegral g) > val2 -> double $ val1 + val2
_ -> Nothing
}
ruleNumbersSuffixesKMG :: Rule
ruleNumbersSuffixesKMG = Rule
{ name = "numbers suffixes (K, M, G)"
, pattern =
[ dimension Numeral
, regex "([kmg])(?=[\\W\\$\x20ac]|$)"
]
, prod = \tokens -> case tokens of
(Token Numeral (NumeralData {TNumeral.value = v}):
Token RegexMatch (GroupMatch (match:_)):
_) -> case Text.toLower match of
"k" -> double $ v * 1e3
"m" -> double $ v * 1e6
"g" -> double $ v * 1e9
_ -> Nothing
_ -> Nothing
}
ruleAPair :: Rule
ruleAPair = Rule
{ name = "a pair"
, pattern =
[ regex "par"
]
, prod = \_ -> integer 2 >>= withGrain 1
}
ruleDozen :: Rule
ruleDozen = Rule
{ name = "dozen"
, pattern =
[ regex "tucet?"
]
, prod = \_ -> integer 12 >>= withGrain 1
}
ruleInteger :: Rule
ruleInteger = Rule
{ name = "integer (0..19)"
, pattern =
[ regex "(ni(s|\x0161)ta|ni(s|\x0161)tica|nula|jedanaest|dvanaest|trinaest|jeda?n(a|u|o(ga?)?)?|dv(i?je)?(a|o)?(ma)?|tri(ma)?|(\x010d|c)etiri|(\x010d|c)etrnaest|petnaest|pet|(s|\x0161)esnaest|(\x0161|s)est|sedamnaest|sedam|osamnaest|osam|devetnaest|devet)"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):_) -> case Text.toLower match of
"ni\353ta" -> integer 0
"ni\353tica" -> integer 0
"nistica" -> integer 0
"nista" -> integer 0
"nula" -> integer 0
"jednoga" -> integer 1
"jedna" -> integer 1
"jednog" -> integer 1
"jednu" -> integer 1
"jedan" -> integer 1
"dvoma" -> integer 2
"dvije" -> integer 2
"dvje" -> integer 2
"dva" -> integer 2
"dvama" -> integer 2
"trima" -> integer 3
"tri" -> integer 3
"\269etiri" -> integer 4
"cetiri" -> integer 4
"pet" -> integer 5
"\353est" -> integer 6
"sedam" -> integer 7
"osam" -> integer 8
"devet" -> integer 9
"jedanaest" -> integer 11
"dvanaest" -> integer 12
"trinaest" -> integer 13
"cetrnaest" -> integer 14
"\269etrnaest" -> integer 14
"petnaest" -> integer 15
"\353esnaest" -> integer 16
"sesnaest" -> integer 16
"sedamnaest" -> integer 17
"osamnaest" -> integer 18
"devetnaest" -> integer 19
_ -> Nothing
_ -> Nothing
}
ruleInteger4 :: Rule
ruleInteger4 = Rule
{ name = "integer 21..99"
, pattern =
[ oneOf [70, 20, 60, 50, 40, 90, 30, 80]
, numberBetween 1 10
]
, prod = \tokens -> case tokens of
(Token Numeral (NumeralData {TNumeral.value = v1}):
Token Numeral (NumeralData {TNumeral.value = v2}):
_) -> double $ v1 + v2
_ -> Nothing
}
ruleInteger2 :: Rule
ruleInteger2 = Rule
{ name = "integer (20..90)"
, pattern =
[ regex "(dvadeset|trideset|(c|\x010d)etrdeset|pedeset|(\x0161|s)esdeset|sedamdeset|osamdeset|devedeset)"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):_) -> case Text.toLower match of
"dvadeset" -> integer 20
"trideset" -> integer 30
"cetrdeset" -> integer 40
"\269etrdeset" -> integer 40
"pedeset" -> integer 50
"sesdeset" -> integer 60
"\353esdeset" -> integer 60
"sedamdeset" -> integer 70
"osamdeset" -> integer 80
"devedeset" -> integer 90
_ -> Nothing
_ -> Nothing
}
ruleNumbers :: Rule
ruleNumbers = Rule
{ name = "numbers 100..999"
, pattern =
[ numberBetween 1 10
, numberWith TNumeral.value (== 100)
, numberBetween 0 100
]
, prod = \tokens -> case tokens of
(Token Numeral (NumeralData {TNumeral.value = v1}):
_:
Token Numeral (NumeralData {TNumeral.value = v2}):
_) -> double $ 100 * v1 + v2
_ -> Nothing
}
ruleNumberDotNumber :: Rule
ruleNumberDotNumber = Rule
{ name = "number dot number"
, pattern =
[ dimension Numeral
, regex "cijela|to(c|\x010d)ka|zarez"
, numberWith TNumeral.grain isNothing
]
, prod = \tokens -> case tokens of
(Token Numeral (NumeralData {TNumeral.value = v1}):
_:
Token Numeral (NumeralData {TNumeral.value = v2}):
_) -> double $ v1 + decimalsToDouble v2
_ -> Nothing
}
ruleIntegerWithThousandsSeparator :: Rule
ruleIntegerWithThousandsSeparator = Rule
{ name = "integer with thousands separator ."
, pattern =
[ regex "(\\d{1,3}(\\.\\d\\d\\d){1,5})"
]
, prod = \tokens -> case tokens of
(Token RegexMatch (GroupMatch (match:_)):
_) -> let fmt = Text.replace (Text.singleton '.') Text.empty match
in parseDouble fmt >>= double
_ -> Nothing
}
ruleMultiply :: Rule
ruleMultiply = Rule
{ name = "compose by multiplication"
, pattern =
[ dimension Numeral
, numberWith TNumeral.multipliable id
]
, prod = \tokens -> case tokens of
(token1:token2:_) -> multiply token1 token2
_ -> Nothing
}
rules :: [Rule]
rules =
[ ruleAPair
, ruleDecimalNumber
, ruleDecimalWithThousandsSeparator
, ruleDozen
, ruleFew
, ruleInteger
, ruleInteger2
, ruleInteger3
, ruleInteger4 , ruleIntegerNumeric
, ruleIntegerWithThousandsSeparator
, ruleMultiply
, ruleNumberDotNumber
, ruleNumbers
, ruleNumbersI
, ruleNumbersPrefixWithNegativeOrMinus
, ruleNumbersSuffixesKMG
, rulePowersOfTen
, ruleSingle
, ruleSum
, ruleTen
]

View File

@ -20,6 +20,7 @@ import qualified Duckling.Ranking.Classifiers.ET as ETClassifiers
import qualified Duckling.Ranking.Classifiers.FR as FRClassifiers
import qualified Duckling.Ranking.Classifiers.GA as GAClassifiers
import qualified Duckling.Ranking.Classifiers.HE as HEClassifiers
import qualified Duckling.Ranking.Classifiers.HR as HRClassifiers
import qualified Duckling.Ranking.Classifiers.ID as IDClassifiers
import qualified Duckling.Ranking.Classifiers.IT as ITClassifiers
import qualified Duckling.Ranking.Classifiers.JA as JAClassifiers
@ -48,6 +49,7 @@ classifiers ET = ETClassifiers.classifiers
classifiers FR = FRClassifiers.classifiers
classifiers GA = GAClassifiers.classifiers
classifiers HE = HEClassifiers.classifiers
classifiers HR = HRClassifiers.classifiers
classifiers ID = IDClassifiers.classifiers
classifiers IT = ITClassifiers.classifiers
classifiers JA = JAClassifiers.classifiers

View File

@ -0,0 +1,22 @@
-- Copyright (c) 2016-present, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
-----------------------------------------------------------------
-- Auto-generated by regenClassifiers
--
-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
-- @generated
-----------------------------------------------------------------
{-# LANGUAGE OverloadedStrings #-}
module Duckling.Ranking.Classifiers.HR (classifiers) where
import Prelude
import Duckling.Ranking.Types
import qualified Data.HashMap.Strict as HashMap
import Data.String
classifiers :: Classifiers
classifiers = HashMap.fromList []

View File

@ -31,6 +31,7 @@ import qualified Duckling.Rules.ET as ETRules
import qualified Duckling.Rules.FR as FRRules
import qualified Duckling.Rules.GA as GARules
import qualified Duckling.Rules.HE as HERules
import qualified Duckling.Rules.HR as HRRules
import qualified Duckling.Rules.ID as IDRules
import qualified Duckling.Rules.IT as ITRules
import qualified Duckling.Rules.JA as JARules
@ -75,6 +76,7 @@ langRules ET = ETRules.rules
langRules FR = FRRules.rules
langRules GA = GARules.rules
langRules HE = HERules.rules
langRules HR = HRRules.rules
langRules ID = IDRules.rules
langRules IT = ITRules.rules
langRules JA = JARules.rules

34
Duckling/Rules/HR.hs Normal file
View File

@ -0,0 +1,34 @@
-- Copyright (c) 2016-present, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
{-# LANGUAGE GADTs #-}
{-# LANGUAGE OverloadedStrings #-}
module Duckling.Rules.HR
( rules
) where
import Duckling.Dimensions.Types
import Duckling.Types
import qualified Duckling.Numeral.HR.Rules as Numeral
rules :: Some Dimension -> [Rule]
rules (This Distance) = []
rules (This Duration) = []
rules (This Numeral) = Numeral.rules
rules (This Email) = []
rules (This AmountOfMoney) = []
rules (This Ordinal) = []
rules (This PhoneNumber) = []
rules (This Quantity) = []
rules (This RegexMatch) = []
rules (This Temperature) = []
rules (This Time) = []
rules (This TimeGrain) = []
rules (This Url) = []
rules (This Volume) = []

View File

@ -46,6 +46,7 @@ library
, Duckling.Rules.FR
, Duckling.Rules.GA
, Duckling.Rules.HE
, Duckling.Rules.HR
, Duckling.Rules.ID
, Duckling.Rules.IT
, Duckling.Rules.JA
@ -76,6 +77,7 @@ library
, Duckling.Ranking.Classifiers.FR
, Duckling.Ranking.Classifiers.GA
, Duckling.Ranking.Classifiers.HE
, Duckling.Ranking.Classifiers.HR
, Duckling.Ranking.Classifiers.IT
, Duckling.Ranking.Classifiers.KO
, Duckling.Ranking.Classifiers.NB
@ -109,6 +111,7 @@ library
, Duckling.Dimensions.FR
, Duckling.Dimensions.GA
, Duckling.Dimensions.HE
, Duckling.Dimensions.HR
, Duckling.Dimensions.ID
, Duckling.Dimensions.IT
, Duckling.Dimensions.JA
@ -232,6 +235,8 @@ library
, Duckling.Numeral.GA.Rules
, Duckling.Numeral.HE.Corpus
, Duckling.Numeral.HE.Rules
, Duckling.Numeral.HR.Corpus
, Duckling.Numeral.HR.Rules
, Duckling.Numeral.ID.Corpus
, Duckling.Numeral.ID.Rules
, Duckling.Numeral.IT.Corpus
@ -551,6 +556,7 @@ test-suite duckling-test
, Duckling.Numeral.FR.Tests
, Duckling.Numeral.GA.Tests
, Duckling.Numeral.HE.Tests
, Duckling.Numeral.HR.Tests
, Duckling.Numeral.ID.Tests
, Duckling.Numeral.IT.Tests
, Duckling.Numeral.JA.Tests

View File

@ -74,6 +74,7 @@ regenClassifiers lang = do
FR -> FRTime.corpus
GA -> GATime.corpus
HE -> (testContext, [])
HR -> (testContext, [])
ID -> (testContext, [])
IT -> ITTime.corpus
JA -> (testContext, [])

View File

@ -0,0 +1,22 @@
-- Copyright (c) 2016-present, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
module Duckling.Numeral.HR.Tests
( tests ) where
import Data.String
import Prelude
import Test.Tasty
import Duckling.Dimensions.Types
import Duckling.Numeral.HR.Corpus
import Duckling.Testing.Asserts
tests :: TestTree
tests = testGroup "HR Tests"
[ makeCorpusTest [This Numeral] corpus
]

View File

@ -21,6 +21,7 @@ import qualified Duckling.Numeral.ET.Tests as ET
import qualified Duckling.Numeral.FR.Tests as FR
import qualified Duckling.Numeral.GA.Tests as GA
import qualified Duckling.Numeral.HE.Tests as HE
import qualified Duckling.Numeral.HR.Tests as HR
import qualified Duckling.Numeral.ID.Tests as ID
import qualified Duckling.Numeral.IT.Tests as IT
import qualified Duckling.Numeral.JA.Tests as JA
@ -49,6 +50,7 @@ tests = testGroup "Numeral Tests"
, FR.tests
, GA.tests
, HE.tests
, HR.tests
, ID.tests
, IT.tests
, JA.tests